Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): isolate Spur cache with a dedicated Redis client (#4545)
Gergő Móricz329377c76650
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 83c4745da..35ef78bb1 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -3,6 +3,8 @@ NUM_WORKERS_PER_QUEUE=8PORT=3002HOST=0.0.0.0REDIS_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379+# Optional dedicated datastore for Spur results, failures and locks; defaults to REDIS_RATE_LIMIT_URL.+# SPUR_REDIS_URL=redis://localhost:6380REDIS_RATE_LIMIT_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379PLAYWRIGHT_MICROSERVICE_URL=http://playwright-service:3000/scrapediff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsindex 893f63256..b92fa8f3d 100644--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -8,6 +8,7 @@ import {scrapeTimeout,} from "../lib";import { redisRateLimitClient } from "../../../services/rate-limiter";+import { redisSpurClient } from "../../../services/spur-redis";import { db } from "../../../db/connection";import * as schema from "../../../db/schema";import { and, desc, eq, gt } from "drizzle-orm";@@ -774,8 +775,7 @@ describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {afterAll(async () => {delete config.USE_DB_AUTHENTICATION;- const keys = await redisRateLimitClient.keys("spur_context:*");- if (keys.length > 0) await redisRateLimitClient.del(...keys);+ await redisSpurClient.del(spurKey("203.0.113.66"), spurKey("203.0.113.67"));});it(@@ -783,11 +783,11 @@ describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {async () => {const ip = "203.0.113.66";// Seed the Spur cache with a suspicious context (active VPN tunnel).- await redisRateLimitClient.set(+ await redisSpurClient.set(spurKey(ip),JSON.stringify({ip,- tunnels: [{ type: "VPN", operator: "PROTON_VPN" }],+ tunnels: [{ anonymous: true, type: "VPN", operator: "PROTON_VPN" }],}),);@@ -818,7 +818,7 @@ describeIf(SPUR_ENABLED)("Keyless free tier — Spur IP reputation", () => {async () => {const ip = "203.0.113.67";// Seed a clean context: datacenter alone is not "suspicious".- await redisRateLimitClient.set(+ await redisSpurClient.set(spurKey(ip),JSON.stringify({ ip, infrastructure: "DATACENTER", risks: [] }),);diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex a12e1c2fc..481893b28 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -219,6 +219,7 @@ const configSchema = z.object({REDIS_URL: z.string().optional(),REDIS_EVICT_URL: z.string().optional(),REDIS_RATE_LIMIT_URL: z.string().optional(),+ SPUR_REDIS_URL: z.string().optional(),NUQ_DATABASE_URL: z.string().optional(),NUQ_DATABASE_URL_LISTEN: z.string().optional(),NUQ_RABBITMQ_URL: z.string().optional(),diff --git a/apps/api/src/lib/spur.ts b/apps/api/src/lib/spur.tsindex 838a27839..c6117f526 100644--- a/apps/api/src/lib/spur.ts+++ b/apps/api/src/lib/spur.ts@@ -12,7 +12,7 @@import { randomUUID } from "crypto";import { isIPv4 } from "net";import { config } from "../config";-import { redisRateLimitClient } from "../services/rate-limiter";+import { redisSpurClient } from "../services/spur-redis";import { logger } from "./logger";const FETCH_TIMEOUT_MS = 5000;@@ -115,10 +115,7 @@ async function readCache(ip: string): Promise<CacheState> {let raw: string | null;let failed: string | null;try {- [raw, failed] = await redisRateLimitClient.mget(- contextKey(ip),- failedKey(ip),- );+ [raw, failed] = await redisSpurClient.mget(contextKey(ip), failedKey(ip));} catch (error) {logger.warn("Spur cache read failed", { ...meta(ip), error });return { state: "miss" };@@ -136,7 +133,7 @@ async function writeCache(ttlSec: number,) {try {- await redisRateLimitClient.set(key, value, "EX", ttlSec);+ await redisSpurClient.set(key, value, "EX", ttlSec);} catch (error) {logger.warn("Spur cache write failed", { ...meta(ip), key, error });}@@ -144,7 +141,7 @@ async function writeCache(async function releaseLock(ip: string, token: string) {try {- await redisRateLimitClient.eval(RELEASE_LOCK_SCRIPT, 1, lockKey(ip), token);+ await redisSpurClient.eval(RELEASE_LOCK_SCRIPT, 1, lockKey(ip), token);} catch (error) {logger.warn("Spur lock release failed", { ...meta(ip), error });}@@ -228,7 +225,7 @@ export async function isKeylessIpSuspicious(ip: string): Promise<boolean> {let locked: boolean;try {locked =- (await redisRateLimitClient.set(+ (await redisSpurClient.set(lockKey(ip),token,"PX",diff --git a/apps/api/src/services/spur-redis.ts b/apps/api/src/services/spur-redis.tsnew file mode 100644index 000000000..d641dbdc1--- /dev/null+++ b/apps/api/src/services/spur-redis.ts@@ -0,0 +1,21 @@+import Redis from "ioredis";+import { config } from "../config";+import { logger } from "../lib/logger";+import { redisRateLimitClient } from "./rate-limiter";++// Keep existing self-hosted configurations working without another datastore.+// Once configured, outages must fail open in Spur, not fall back to an old cache.+export const redisSpurClient = config.SPUR_REDIS_URL+ ? new Redis(config.SPUR_REDIS_URL, {+ enableAutoPipelining: true,+ enableOfflineQueue: false,+ maxRetriesPerRequest: 1,+ commandTimeout: 1000,+ })+ : redisRateLimitClient;++if (config.SPUR_REDIS_URL) {+ redisSpurClient.on("error", () => {+ logger.warn("Spur Redis connection error", { module: "spur-redis" });+ });+}