Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(citations): chunk URL lookups so answers with many citations are not dropped (#738)
Gökhan Geyik59b33a0328f7
diff --git a/server/src/lib/citation-rows.js b/server/src/lib/citation-rows.jsindex 9d8cddda2..d8622b5c9 100644--- a/server/src/lib/citation-rows.js+++ b/server/src/lib/citation-rows.js@@ -73,6 +73,43 @@ export function normalizeCitations(rawCitations) {return out;}+/**+ * URLs looked up per request.+ *+ * A PostgREST `.in()` filter is spelled out in the query string, so the whole+ * list travels in the request line — which has a length limit that a plain+ * insert does not. The first version sent every URL of an answer at once and+ * silently lost the answers that most needed storing: 228 of 174,466 in the+ * first production backfill, averaging 45 citations and 16 KB of URL text+ * each, against 12 citations and 1.2 KB for the ones that succeeded. The+ * failure surfaced as a logged error and a skipped answer, not as a crash.+ *+ * Sized so that even pathological URLs stay well inside the limit: 100 URLs+ * at the observed 2,048-character ceiling is roughly 200 KB, and the longest+ * single answer seen carries 191 citations.+ */+const URL_LOOKUP_CHUNK = 100;++function chunk(items, size) {+ const out = [];+ for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));+ return out;+}++/** `select id, url where url in (...)`, split across as many requests as it takes. */+async function selectUrlIds(urls, label) {+ const ids = new Map();+ for (const batch of chunk(urls, URL_LOOKUP_CHUNK)) {+ const { data, error } = await supabaseAdmin+ .from('citation_urls')+ .select('id, url')+ .in('url', batch);+ if (error) throw new Error(`citation url ${label}: ${error.message}`);+ for (const row of data ?? []) ids.set(row.url, row.id);+ }+ return ids;+}+/*** Resolve URLs to dictionary ids, inserting the ones not seen before.*@@ -93,16 +130,12 @@ async function resolveUrlIds(entries) {const urls = [...byUrl.keys()];if (urls.length === 0) return new Map();- const { data: known, error: knownErr } = await supabaseAdmin- .from('citation_urls')- .select('id, url')- .in('url', urls);- if (knownErr) throw new Error(`citation url lookup: ${knownErr.message}`);-- const ids = new Map((known ?? []).map((row) => [row.url, row.id]));+ const ids = await selectUrlIds(urls, 'lookup');const missing = urls.filter((url) => !ids.has(url));if (missing.length > 0) {+ // The insert itself carries its payload in the body, not the query string,+ // so it needs no chunking — only the `.in()` reads around it do.const { error: insertErr } = await supabaseAdmin.from('citation_urls').upsert(missing.map((url) => {const entry = byUrl.get(url);@@ -112,12 +145,7 @@ async function resolveUrlIds(entries) {);if (insertErr) throw new Error(`citation url insert: ${insertErr.message}`);- const { data: added, error: addedErr } = await supabaseAdmin- .from('citation_urls')- .select('id, url')- .in('url', missing);- if (addedErr) throw new Error(`citation url re-read: ${addedErr.message}`);- for (const row of added ?? []) ids.set(row.url, row.id);+ for (const [url, id] of await selectUrlIds(missing, 're-read')) ids.set(url, id);}return ids;diff --git a/server/src/lib/citation-rows.test.js b/server/src/lib/citation-rows.test.jsindex 21986f8db..25b65ea2c 100644--- a/server/src/lib/citation-rows.test.js+++ b/server/src/lib/citation-rows.test.js@@ -108,3 +108,87 @@ describe('normalizeCitations', () => {expect(rows.map((r) => r.position)).toEqual([0, 1]);});});++/**+ * URL lookups are chunked because a PostgREST `.in()` filter travels in the+ * query string (#732).+ *+ * The first production backfill lost 228 of 174,466 answers to this — the ones+ * carrying the most citations, averaging 16 KB of URL text against 1.2 KB for+ * the answers that succeeded. The failure was logged and the answer skipped,+ * so the totals looked healthy. These tests pin the batching rather than the+ * limit, since the limit is the server's to enforce.+ */+describe('persistCitationRows url lookups', () => {+ /** Records every `.in()` batch so the test can assert on their sizes. */+ function makeClient({ existing = new Map() } = {}) {+ const batches = [];+ let nextId = existing.size + 1;++ const client = {+ from(table) {+ if (table === 'citation_urls') {+ return {+ select: () => ({+ in: (_col, urls) => {+ batches.push(urls.length);+ return Promise.resolve({+ data: urls+ .filter((u) => existing.has(u))+ .map((u) => ({ id: existing.get(u), url: u })),+ error: null,+ });+ },+ }),+ upsert: (rows) => {+ for (const row of rows) if (!existing.has(row.url)) existing.set(row.url, nextId++);+ return Promise.resolve({ error: null });+ },+ };+ }+ return { upsert: () => Promise.resolve({ error: null }) };+ },+ };+ return { client, batches };+ }++ async function run(citations, opts) {+ const { client, batches } = makeClient(opts);+ vi.resetModules();+ vi.doMock('../config/supabase.js', () => ({ default: client }));+ const { persistCitationRows } = await import('./citation-rows.js');+ const written = await persistCitationRows({+ promptResultId: 'r1',+ brandId: 'b1',+ createdAt: '2026-08-19T00:00:00Z',+ citations,+ });+ return { written, batches };+ }++ it('splits a large URL set into bounded batches', async () => {+ const citations = Array.from({ length: 250 }, (_, i) => ({ url: `https://a.com/${i}` }));+ const { written, batches } = await run(citations);++ expect(written).toBe(250);+ // 250 unknown URLs: three batches to look them up, three to read them back.+ expect(batches).toEqual([100, 100, 50, 100, 100, 50]);+ expect(Math.max(...batches)).toBeLessThanOrEqual(100);+ });++ it('stores every citation of an answer far larger than one batch', async () => {+ // The largest answer in production carries 191 citations.+ const citations = Array.from({ length: 191 }, (_, i) => ({ url: `https://b.com/${i}` }));+ const { written } = await run(citations);+ expect(written).toBe(191);+ });++ it('makes no second round trip when every URL is already known', async () => {+ const existing = new Map(Array.from({ length: 120 }, (_, i) => [`https://c.com/${i}`, i + 1]));+ const citations = Array.from({ length: 120 }, (_, i) => ({ url: `https://c.com/${i}` }));+ const { written, batches } = await run(citations, { existing });++ expect(written).toBe(120);+ expect(batches).toEqual([100, 20]); // lookup only — nothing to insert+ });+});diff --git a/server/src/scripts/backfill-citations.js b/server/src/scripts/backfill-citations.jsindex 0e95f7d79..f7466e388 100644--- a/server/src/scripts/backfill-citations.js+++ b/server/src/scripts/backfill-citations.js@@ -72,6 +72,7 @@ async function main() {let answers = 0;let citations = 0;let empty = 0;+ const failed = [];console.log(`Backfilling citations — batch ${BATCH}, pause ${PAUSE_MS}ms` +@@ -83,6 +84,7 @@ async function main() {if (batch.length === 0) break;for (const row of batch) {+ const expected = Array.isArray(row.citations) ? row.citations.length : 0;const written = await persistCitationRows({promptResultId: row.id,brandId: row.brand_id,@@ -91,7 +93,13 @@ async function main() {});answers += 1;citations += written;- if (written === 0) empty += 1;+ if (expected === 0) empty += 1;+ // An answer that had citations and stored none wrote nothing at all:+ // persistCitationRows is best-effort and reports failure by returning 0.+ // Counting those separately is what makes a systematic fault visible —+ // the first production run lost 228 answers this way and the totals+ // alone looked healthy.+ else if (written === 0) failed.push(row.id);}const last = batch[batch.length - 1];@@ -99,7 +107,8 @@ async function main() {const elapsed = Math.round((Date.now() - startedAt) / 1000);console.log(- ` ${answers} answers · ${citations} citations · ${empty} with none · ${elapsed}s · at ${last.created_at}`,+ ` ${answers} answers · ${citations} citations · ${empty} with none · ` ++ `${failed.length} failed · ${elapsed}s · at ${last.created_at}`,);if (batch.length < BATCH) break;@@ -111,6 +120,17 @@ async function main() {`\nDone. ${answers} answers scanned, ${citations} citation rows written, ` +`${empty} answers had none, ${elapsed}s.`,);++ if (failed.length > 0) {+ // Loud on purpose. These answers have citations and stored none, so the+ // table is short by however many they carried — and re-running the script+ // is the fix, since everything else is skipped as already present.+ console.error(+ `\n${failed.length} answer(s) had citations but stored none. Re-run to retry them.`,+ );+ console.error(`First few: ${failed.slice(0, 10).join(', ')}`);+ process.exitCode = 1;+ }}main().catch((err) => {