Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
address review round 4: replace KEYS scan with candidate-key MGET in keyless snips
Karan Lokchandani0b312c63be8a
diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex b258532a0..7ba4abb9a 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -1,3 +1,4 @@+import os from "os";import { config } from "../../../config";import { describeIf, itIf, TEST_PRODUCTION } from "../lib";import { creditUsage, idmux, researchRaw } from "./lib";@@ -49,12 +50,29 @@ function keylessHeaders(forwardedIp: string): Record<string, string> {: {};}+// Without the trusted-proxy secret the server keys keyless requests on the+// real client IP, which for these tests is the loopback or a local interface+// address. Snapshotting only those candidate keys (plain MGET) avoids the+// blocking KEYS scan on the shared rate-limit Redis, and stays deterministic+// under concurrency: every parallel no-secret snip bumps the same client-IP+// key, so the diff below still identifies exactly one bumped candidate.+function candidateKeylessIps(): string[] {+ const ips = new Set<string>(["127.0.0.1"]);+ for (const addrs of Object.values(os.networkInterfaces())) {+ for (const addr of addrs ?? []) {+ if (addr.family === "IPv4") ips.add(addr.address);+ }+ }+ return [...ips];+}+async function snapshotKeylessCounts(): Promise<Map<string, number>> {- const keys = await redisRateLimitClient.keys("keyless_requests:*");+ const candidates = candidateKeylessIps();+ const values = await redisRateLimitClient.mget(+ candidates.map(ip => `keyless_requests:${ip}`),+ );const counts = new Map<string, number>();- for (const key of keys) {- counts.set(key, Number((await redisRateLimitClient.get(key)) ?? "0"));- }+ candidates.forEach((ip, i) => counts.set(ip, Number(values[i] ?? "0")));return counts;}@@ -65,8 +83,8 @@ async function resolveKeylessIp(if (KEYLESS_PROXY_SECRET) return forwardedIp;const after = await snapshotKeylessCounts();const bumped = [...after.entries()]- .filter(([key, count]) => count > (before.get(key) ?? 0))- .map(([key]) => key.slice("keyless_requests:".length));+ .filter(([ip, count]) => count > (before.get(ip) ?? 0))+ .map(([ip]) => ip);expect(bumped.length).toBe(1);return bumped[0];}