Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(citations): page the already-stored lookup past PostgREST's 1000-row cap (#740)

Gökhan Geyik9da82f7434c7
diff --git a/server/src/lib/citation-rows.js b/server/src/lib/citation-rows.js
index d8622b5c9..1cf161426 100644
--- a/server/src/lib/citation-rows.js
+++ b/server/src/lib/citation-rows.js
@@ -159,7 +159,10 @@ async function resolveUrlIds(entries) {
* so anything missed is recoverable by re-running the backfill over the
* affected window.
*
- * @returns {Promise<number>} rows written (0 when there is nothing to write, or on failure)
+ * @returns {Promise<number|null>} rows actually inserted, or null if the write
+ * failed. Zero and null are deliberately different answers: zero means every
+ * row was already there, null means nothing was stored and the caller should
+ * retry. Conflating them is how a broken run reads as a quiet one.
*/
export async function persistCitationRows({ promptResultId, brandId, createdAt, citations }) {
const entries = normalizeCitations(citations);
@@ -183,17 +186,23 @@ export async function persistCitationRows({ promptResultId, brandId, createdAt,
// Idempotent on (prompt_result_id, position): a retried insert, a
// re-delivered webhook, or a re-run of the backfill over the same answer
// adds nothing and errors on nothing.
- const { error } = await supabaseAdmin
+ //
+ // `.select()` so the return value is what was actually inserted rather
+ // than what was attempted. The difference matters: a backfill run once
+ // reported thousands of citations written while the table did not move,
+ // because every one of them was silently discarded as a duplicate.
+ const { data, error } = await supabaseAdmin
.from('prompt_result_citations')
- .upsert(rows, { onConflict: 'prompt_result_id,position', ignoreDuplicates: true });
+ .upsert(rows, { onConflict: 'prompt_result_id,position', ignoreDuplicates: true })
+ .select('position');
if (error) throw new Error(error.message);
- return rows.length;
+ return (data ?? []).length;
} catch (err) {
logger.error(
{ err, promptResultId, brandId, citationCount: entries.length },
'[citations] failed to persist citation rows',
);
- return 0;
+ return null;
}
}
diff --git a/server/src/lib/citation-rows.test.js b/server/src/lib/citation-rows.test.js
index 25b65ea2c..5b192c1d0 100644
--- a/server/src/lib/citation-rows.test.js
+++ b/server/src/lib/citation-rows.test.js
@@ -146,7 +146,14 @@ describe('persistCitationRows url lookups', () => {
},
};
}
- return { upsert: () => Promise.resolve({ error: null }) };
+ // prompt_result_citations: `.upsert().select()` returns the rows that
+ // were actually inserted, which is what persistCitationRows counts.
+ return {
+ upsert: (rows) => ({
+ select: () =>
+ Promise.resolve({ data: rows.map((r) => ({ position: r.position })), error: null }),
+ }),
+ };
},
};
return { client, batches };
@@ -192,3 +199,84 @@ describe('persistCitationRows url lookups', () => {
expect(batches).toEqual([100, 20]); // lookup only — nothing to insert
});
});
+
+/**
+ * Zero and null are different answers (#732).
+ *
+ * A backfill run once reported thousands of citations written while the table
+ * did not move, because every row was discarded as a duplicate and the count
+ * reported what had been attempted. The caller has to be able to tell "already
+ * there" from "nothing stored, retry".
+ */
+describe('persistCitationRows return contract', () => {
+ async function withClient(client, citations) {
+ vi.resetModules();
+ vi.doMock('../config/supabase.js', () => ({ default: client }));
+ const { persistCitationRows } = await import('./citation-rows.js');
+ return persistCitationRows({
+ promptResultId: 'r1',
+ brandId: 'b1',
+ createdAt: '2026-08-19T00:00:00Z',
+ citations,
+ });
+ }
+
+ const urlTable = (inserted) => ({
+ select: () => ({
+ in: (_c, urls) =>
+ Promise.resolve({ data: urls.map((u, i) => ({ id: i + 1, url: u })), error: null }),
+ }),
+ upsert: () => Promise.resolve({ error: null }),
+ _inserted: inserted,
+ });
+
+ it('reports how many rows were actually inserted', async () => {
+ const client = {
+ from: (t) =>
+ t === 'citation_urls'
+ ? urlTable()
+ : { upsert: (rows) => ({ select: () => Promise.resolve({ data: rows, error: null }) }) },
+ };
+ expect(await withClient(client, [{ url: 'https://a.com/1' }, { url: 'https://a.com/2' }])).toBe(
+ 2,
+ );
+ });
+
+ it('returns 0 — not a failure — when every row was already present', async () => {
+ const client = {
+ from: (t) =>
+ t === 'citation_urls'
+ ? urlTable()
+ : { upsert: () => ({ select: () => Promise.resolve({ data: [], error: null }) }) },
+ };
+ expect(await withClient(client, [{ url: 'https://a.com/1' }])).toBe(0);
+ });
+
+ it('returns null when the write fails', async () => {
+ const client = {
+ from: (t) =>
+ t === 'citation_urls'
+ ? urlTable()
+ : {
+ upsert: () => ({
+ select: () => Promise.resolve({ data: null, error: { message: 'boom' } }),
+ }),
+ },
+ };
+ expect(await withClient(client, [{ url: 'https://a.com/1' }])).toBeNull();
+ });
+
+ it('returns null when the URL lookup fails', async () => {
+ const client = {
+ from: () => ({
+ select: () => ({ in: () => Promise.resolve({ data: null, error: { message: 'boom' } }) }),
+ }),
+ };
+ expect(await withClient(client, [{ url: 'https://a.com/1' }])).toBeNull();
+ });
+
+ it('returns 0 for an answer with no storable citations', async () => {
+ const client = { from: () => ({}) };
+ expect(await withClient(client, [{ url: '/relative' }])).toBe(0);
+ });
+});
diff --git a/server/src/scripts/backfill-citations.js b/server/src/scripts/backfill-citations.js
index 4ae0be146..aa7d37e50 100644
--- a/server/src/scripts/backfill-citations.js
+++ b/server/src/scripts/backfill-citations.js
@@ -45,6 +45,12 @@ const FORCE = process.argv.includes('--force');
*/
const DONE_LOOKUP_CHUNK = 200;
+/**
+ * PostgREST's own default page size. Reading in exactly this size means a
+ * short page is an unambiguous end-of-results signal.
+ */
+const PAGE_LIMIT = 1000;
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
/**
@@ -88,8 +94,16 @@ async function fetchBatch(cursor) {
* The upsert makes re-running safe but not cheap: it skips the write and still
* pays for resolving every URL of every answer first. On the full history that
* is ~18 answers a second, so a re-run to repair a few hundred answers costs
- * most of three hours. One extra query per page — ids only, no jsonb — turns
- * that into minutes.
+ * most of three hours. One lookup per page — ids only, no jsonb — turns that
+ * into minutes.
+ *
+ * Paginated, because the answer is one row per *citation* rather than per
+ * answer: 200 answers matched 1,591 rows in production, and PostgREST caps an
+ * un-paginated select at 1,000. The first version read only that first page,
+ * so every answer past it looked unstored and was reprocessed — the upsert
+ * then discarded the writes as duplicates, and the run reported thousands of
+ * citations written while the table did not move at all. Same trap as #714
+ * and #716.
*/
async function alreadyStored(ids) {
if (ids.length === 0) return new Set();
@@ -97,12 +111,20 @@ async function alreadyStored(ids) {
const done = new Set();
for (let i = 0; i < ids.length; i += DONE_LOOKUP_CHUNK) {
const slice = ids.slice(i, i + DONE_LOOKUP_CHUNK);
- const { data, error } = await supabaseAdmin
- .from('prompt_result_citations')
- .select('prompt_result_id')
- .in('prompt_result_id', slice);
- if (error) throw new Error(`stored lookup: ${error.message}`);
- for (const row of data ?? []) done.add(row.prompt_result_id);
+
+ for (let offset = 0; ; offset += PAGE_LIMIT) {
+ const { data, error } = await supabaseAdmin
+ .from('prompt_result_citations')
+ .select('prompt_result_id')
+ .in('prompt_result_id', slice)
+ .order('prompt_result_id', { ascending: true })
+ .order('position', { ascending: true })
+ .range(offset, offset + PAGE_LIMIT - 1);
+ if (error) throw new Error(`stored lookup: ${error.message}`);
+
+ for (const row of data ?? []) done.add(row.prompt_result_id);
+ if ((data ?? []).length < PAGE_LIMIT) break;
+ }
}
return done;
}
@@ -159,14 +181,14 @@ async function main() {
citations: raw,
});
answers += 1;
- citations += written;
+ citations += written ?? 0;
+ // Three outcomes, kept apart on purpose. No storable citations is normal.
+ // A failure is not, and it is reported by returning null rather than 0 —
+ // 0 legitimately means "every row was already there", which is what
+ // `--force` produces on every answer it revisits. The first production
+ // run lost 228 answers to a fault the totals alone made look healthy.
if (expected === 0) empty += 1;
- // An answer that had citations and stored none wrote nothing at all:
- // persistCitationRows is best-effort and reports failure by returning 0.
- // Counting those separately is what makes a systematic fault visible —
- // the first production run lost 228 answers this way and the totals
- // alone looked healthy.
- else if (written === 0) failed.push(row.id);
+ else if (written === null) failed.push(row.id);
}
const last = batch[batch.length - 1];