Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(traffic): wait a few minutes before reading a failed bot range list again (#1251)

Cihand101993336cb
diff --git a/server/src/lib/ai-bots.js b/server/src/lib/ai-bots.js
index 434a11f76..72f387ca7 100644
--- a/server/src/lib/ai-bots.js
+++ b/server/src/lib/ai-bots.js
@@ -11,7 +11,9 @@
*
* The ranges are fetched on first use and again once a day, and kept in
* memory. A failed refresh keeps the last list; with none ever read, the
- * bot's requests can't be verified and aren't counted.
+ * bot's requests can't be verified and aren't counted, and the list isn't
+ * tried again for a few minutes. Requests that need a list being read wait
+ * for that one read.
*
* The list is versioned like ai-sources.json: each change adds an entry to
* `changes` and bumps `version`.
@@ -96,33 +98,57 @@ export function readRangeFile(json) {
return list.map((p) => parsePrefix(p?.ipv4Prefix ?? p?.ipv6Prefix)).filter(Boolean);
}
-const cache = new Map();
-
/**
- * A range file's prefixes: from memory while fresh, read again after a day.
- * A failed read keeps what was read before; null when nothing ever was.
+ * After a failed read with no list to fall back on, the next try waits this
+ * long: a batch's requests for that bot would otherwise each wait out the
+ * fetch's timeout, one after another (#1179's review).
*/
-export async function rangesFor(url, { fetchImpl = fetch, now = Date.now() } = {}) {
- const hit = cache.get(url);
- if (hit && now - hit.at < RANGES_TTL_MS) return hit.prefixes;
+export const RANGES_RETRY_MS = 5 * 60 * 1000;
+
+/** A failed refresh keeps the last list this long before the next try. */
+const STALE_KEEP_MS = 60 * 60 * 1000;
+
+/** url → { until, prefixes }: prefixes null after a failed first read. */
+const cache = new Map();
+/** url → the read under way, which concurrent requests share. */
+const pending = new Map();
+
+async function readRanges(url, hit, fetchImpl, now) {
try {
const res = await fetchImpl(url, { signal: AbortSignal.timeout(10_000) });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const prefixes = readRangeFile(await res.json());
if (prefixes.length === 0) throw new Error('no prefixes');
- cache.set(url, { at: now, prefixes });
+ cache.set(url, { until: now + RANGES_TTL_MS, prefixes });
return prefixes;
} catch (err) {
logger.warn({ err, url }, '[ai-bots] range list not refreshed');
- // Keep the last list for another hour before trying again.
- if (hit) cache.set(url, { at: now - RANGES_TTL_MS + 60 * 60 * 1000, prefixes: hit.prefixes });
- return hit?.prefixes ?? null;
+ const prefixes = hit?.prefixes ?? null;
+ cache.set(url, { until: now + (prefixes ? STALE_KEEP_MS : RANGES_RETRY_MS), prefixes });
+ return prefixes;
+ }
+}
+
+/**
+ * A range file's prefixes: from memory while fresh, read again after a day.
+ * A failed read keeps what was read before; null when nothing ever was, and
+ * then until RANGES_RETRY_MS has passed. One read at a time per file.
+ */
+export async function rangesFor(url, { fetchImpl = fetch, now = Date.now() } = {}) {
+ const hit = cache.get(url);
+ if (hit && now < hit.until) return hit.prefixes;
+ let read = pending.get(url);
+ if (!read) {
+ read = readRanges(url, hit, fetchImpl, now).finally(() => pending.delete(url));
+ pending.set(url, read);
}
+ return read;
}
/** Forgets every range list (tests). */
export function clearRangeCache() {
cache.clear();
+ pending.clear();
}
/**
diff --git a/server/src/lib/ai-bots.test.js b/server/src/lib/ai-bots.test.js
index 25d67ca86..0a55ddd85 100644
--- a/server/src/lib/ai-bots.test.js
+++ b/server/src/lib/ai-bots.test.js
@@ -1,6 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import {
AI_BOTS,
+ RANGES_RETRY_MS,
RANGES_TTL_MS,
clearRangeCache,
inPrefixes,
@@ -84,6 +85,33 @@ describe('rangesFor', () => {
const fetchImpl = vi.fn().mockRejectedValue(new Error('offline'));
expect(await rangesFor('https://x/b.json', { fetchImpl, now: 0 })).toBeNull();
});
+
+ // gkhngyk on #1179: a list that failed to load was fetched again for every
+ // request with that bot's user agent, each waiting out the timeout.
+ it('waits a few minutes after a failed first read before trying again', async () => {
+ const fetchImpl = vi.fn().mockRejectedValue(new Error('timeout'));
+ for (const now of [0, 1_000, RANGES_RETRY_MS - 1]) {
+ expect(await rangesFor('https://x/c.json', { fetchImpl, now })).toBeNull();
+ }
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+
+ fetchImpl.mockResolvedValue(json({ prefixes: [{ ipv4Prefix: '1.2.3.0/24' }] }));
+ const later = await rangesFor('https://x/c.json', { fetchImpl, now: RANGES_RETRY_MS });
+ expect(later).toHaveLength(1);
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ });
+
+ it('shares one read among the requests waiting for it', async () => {
+ let answer;
+ const fetchImpl = vi.fn(() => new Promise((resolve) => (answer = resolve)));
+ const reads = [1, 2, 3].map(() => rangesFor('https://x/d.json', { fetchImpl, now: 0 }));
+ answer(json({ prefixes: [{ ipv4Prefix: '1.2.3.0/24' }] }));
+ const lists = await Promise.all(reads);
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ expect(lists[0]).toHaveLength(1);
+ expect(lists[1]).toBe(lists[0]);
+ expect(lists[2]).toBe(lists[0]);
+ });
});
describe('verifyBotRequest', () => {