Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): isolate Spur cache with a dedicated Redis client (#4545)

Gergő Móricz329377c76650
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 83c4745da..35ef78bb1 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -3,6 +3,8 @@ NUM_WORKERS_PER_QUEUE=8
PORT=3002
HOST=0.0.0.0
REDIS_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379
+# Optional dedicated datastore for Spur results, failures and locks; defaults to REDIS_RATE_LIMIT_URL.
+# SPUR_REDIS_URL=redis://localhost:6380
REDIS_RATE_LIMIT_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379
PLAYWRIGHT_MICROSERVICE_URL=http://playwright-service:3000/scrape
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.ts
index 893f63256..b92fa8f3d 100644
--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts
+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts
@@ -8,6 +8,7 @@ import {
scrapeTimeout,
} from "../lib";
import { redisRateLimitClient } from "../../../services/rate-limiter";
+import { redisSpurClient } from "../../../services/spur-redis";
import { db } from "../../../db/connection";
import * as schema from "../../../db/schema";
import { and, desc, eq, gt } from "drizzle-orm";
@@ -774,8 +775,7 @@ describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {
afterAll(async () => {
delete config.USE_DB_AUTHENTICATION;
- const keys = await redisRateLimitClient.keys("spur_context:*");
- if (keys.length > 0) await redisRateLimitClient.del(...keys);
+ await redisSpurClient.del(spurKey("203.0.113.66"), spurKey("203.0.113.67"));
});
it(
@@ -783,11 +783,11 @@ describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {
async () => {
const ip = "203.0.113.66";
// Seed the Spur cache with a suspicious context (active VPN tunnel).
- await redisRateLimitClient.set(
+ await redisSpurClient.set(
spurKey(ip),
JSON.stringify({
ip,
- tunnels: [{ type: "VPN", operator: "PROTON_VPN" }],
+ tunnels: [{ anonymous: true, type: "VPN", operator: "PROTON_VPN" }],
}),
);
@@ -818,7 +818,7 @@ describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {
async () => {
const ip = "203.0.113.67";
// Seed a clean context: datacenter alone is not "suspicious".
- await redisRateLimitClient.set(
+ await redisSpurClient.set(
spurKey(ip),
JSON.stringify({ ip, infrastructure: "DATACENTER", risks: [] }),
);
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index a12e1c2fc..481893b28 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -219,6 +219,7 @@ const configSchema = z.object({
REDIS_URL: z.string().optional(),
REDIS_EVICT_URL: z.string().optional(),
REDIS_RATE_LIMIT_URL: z.string().optional(),
+ SPUR_REDIS_URL: z.string().optional(),
NUQ_DATABASE_URL: z.string().optional(),
NUQ_DATABASE_URL_LISTEN: z.string().optional(),
NUQ_RABBITMQ_URL: z.string().optional(),
diff --git a/apps/api/src/lib/spur.ts b/apps/api/src/lib/spur.ts
index 838a27839..c6117f526 100644
--- a/apps/api/src/lib/spur.ts
+++ b/apps/api/src/lib/spur.ts
@@ -12,7 +12,7 @@
import { randomUUID } from "crypto";
import { isIPv4 } from "net";
import { config } from "../config";
-import { redisRateLimitClient } from "../services/rate-limiter";
+import { redisSpurClient } from "../services/spur-redis";
import { logger } from "./logger";
const FETCH_TIMEOUT_MS = 5000;
@@ -115,10 +115,7 @@ async function readCache(ip: string): Promise<CacheState> {
let raw: string | null;
let failed: string | null;
try {
- [raw, failed] = await redisRateLimitClient.mget(
- contextKey(ip),
- failedKey(ip),
- );
+ [raw, failed] = await redisSpurClient.mget(contextKey(ip), failedKey(ip));
} catch (error) {
logger.warn("Spur cache read failed", { ...meta(ip), error });
return { state: "miss" };
@@ -136,7 +133,7 @@ async function writeCache(
ttlSec: number,
) {
try {
- await redisRateLimitClient.set(key, value, "EX", ttlSec);
+ await redisSpurClient.set(key, value, "EX", ttlSec);
} catch (error) {
logger.warn("Spur cache write failed", { ...meta(ip), key, error });
}
@@ -144,7 +141,7 @@ async function writeCache(
async function releaseLock(ip: string, token: string) {
try {
- await redisRateLimitClient.eval(RELEASE_LOCK_SCRIPT, 1, lockKey(ip), token);
+ await redisSpurClient.eval(RELEASE_LOCK_SCRIPT, 1, lockKey(ip), token);
} catch (error) {
logger.warn("Spur lock release failed", { ...meta(ip), error });
}
@@ -228,7 +225,7 @@ export async function isKeylessIpSuspicious(ip: string): Promise<boolean> {
let locked: boolean;
try {
locked =
- (await redisRateLimitClient.set(
+ (await redisSpurClient.set(
lockKey(ip),
token,
"PX",
diff --git a/apps/api/src/services/spur-redis.ts b/apps/api/src/services/spur-redis.ts
new file mode 100644
index 000000000..d641dbdc1
--- /dev/null
+++ b/apps/api/src/services/spur-redis.ts
@@ -0,0 +1,21 @@
+import Redis from "ioredis";
+import { config } from "../config";
+import { logger } from "../lib/logger";
+import { redisRateLimitClient } from "./rate-limiter";
+
+// Keep existing self-hosted configurations working without another datastore.
+// Once configured, outages must fail open in Spur, not fall back to an old cache.
+export const redisSpurClient = config.SPUR_REDIS_URL
+ ? new Redis(config.SPUR_REDIS_URL, {
+ enableAutoPipelining: true,
+ enableOfflineQueue: false,
+ maxRetriesPerRequest: 1,
+ commandTimeout: 1000,
+ })
+ : redisRateLimitClient;
+
+if (config.SPUR_REDIS_URL) {
+ redisSpurClient.on("error", () => {
+ logger.warn("Spur Redis connection error", { module: "spur-redis" });
+ });
+}