Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(server): page prompt_visibility_summaries reads past the 1000-row cap (#1096)

Cihan3b2fe2783393
diff --git a/server/src/lib/pulse/metrics.js b/server/src/lib/pulse/metrics.js
index 94fcaed98..33bbe01b8 100644
--- a/server/src/lib/pulse/metrics.js
+++ b/server/src/lib/pulse/metrics.js
@@ -2,6 +2,7 @@ import supabaseAdmin from '../../config/supabase.js';
import { computeAiVisibilityScore } from '../../config/visibility-score.js';
import { chunkIds, selectInChunks } from '../chunked-in.js';
import { logger } from '../logger.js';
+import { readAllRpcRows } from '../rpc-pages.js';
import { resolve } from '../../config/action-engine.js';
/**
@@ -104,13 +105,30 @@ export function isTransientDbError(message) {
return /statement timeout|canceling statement|deadlock detected/i.test(message || '');
}
-async function rpc(name, params, attempt = 0) {
- const { data, error } = await supabaseAdmin.rpc(name, params);
+function rpc(name, params) {
+ return retried(name, () => supabaseAdmin.rpc(name, params));
+}
+
+/**
+ * Every row of a set-returning RPC, in `key` order (see lib/rpc-pages.js):
+ * one response stops at 1000 rows. A timeout on any page retries the read
+ * from its first page.
+ */
+function rpcRows(name, params, key) {
+ return retried(name, () =>
+ readAllRpcRows(name, (from, to) =>
+ supabaseAdmin.rpc(name, params).order(key, { ascending: true }).range(from, to),
+ ),
+ );
+}
+
+async function retried(name, read, attempt = 0) {
+ const { data, error } = await read();
if (error) {
if (attempt < RPC_RETRIES && isTransientDbError(error.message)) {
logger.warn({ rpc: name, attempt: attempt + 1 }, 'pulse rpc timed out, retrying');
await new Promise((r) => setTimeout(r, RPC_RETRY_DELAY_MS * (attempt + 1)));
- return rpc(name, params, attempt + 1);
+ return retried(name, read, attempt + 1);
}
throw new Error(`${name} failed: ${error.message}`);
}
@@ -198,13 +216,22 @@ async function competitorRates(brandId, from, to, days = null) {
return { brandRate, competitors };
}
-/** Per-prompt AI Visibility Score over a window (movers highlight). */
-async function promptScores(brandId, from, to) {
- const rows = await rpc('prompt_visibility_summaries', {
- p_brand_id: brandId,
- p_date_from: from.toISOString(),
- p_date_to: to.toISOString(),
- });
+/**
+ * Per-prompt AI Visibility Score over a window (movers highlight). One row per
+ * prompt, so a brand past 1000 prompts needs more than one page.
+ *
+ * Exported for the test that pins the paging behaviour.
+ */
+export async function promptScores(brandId, from, to) {
+ const rows = await rpcRows(
+ 'prompt_visibility_summaries',
+ {
+ p_brand_id: brandId,
+ p_date_from: from.toISOString(),
+ p_date_to: to.toISOString(),
+ },
+ 'prompt_id',
+ );
const map = new Map();
for (const row of rows ?? []) {
if (!row.runs) continue;
diff --git a/server/src/lib/pulse/prompt-scores.test.js b/server/src/lib/pulse/prompt-scores.test.js
new file mode 100644
index 000000000..9c5c83807
--- /dev/null
+++ b/server/src/lib/pulse/prompt-scores.test.js
@@ -0,0 +1,133 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+// metrics.js reaches the admin client at import time, whose config hard-exits
+// without SUPABASE_* env (as in CI). Its `rpc` is replaced per test below.
+vi.mock('../../config/supabase.js', () => ({ default: { rpc: vi.fn(), from: vi.fn() } }));
+vi.mock('../logger.js', () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+import supabaseAdmin from '../../config/supabase.js';
+import { promptScores } from './metrics.js';
+
+const MAX_ROWS = 1000;
+const uuid = (n) => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`;
+const summary = (n) => ({
+ prompt_id: uuid(n),
+ runs: 4,
+ mention_answers: 2,
+ citation_answers: 1,
+ position_factor: 0.5,
+});
+
+/**
+ * PostgREST in miniature for one RPC: rows sorted by `.order()`, sliced by
+ * `.range()`, and never more than max_rows in one response. An un-ranged call
+ * gets the first 1000 rows and no sign that more exist — the defect itself.
+ */
+function serve(rows, { failFirst = null } = {}) {
+ const calls = [];
+ supabaseAdmin.rpc.mockImplementation((name, params) => {
+ const call = { name, params, order: null, range: null };
+ calls.push(call);
+ const respond = () => {
+ if (failFirst && calls.indexOf(call) === 0) return { data: null, error: failFirst };
+ const sorted = call.order
+ ? [...rows].sort((a, b) => a[call.order].localeCompare(b[call.order]))
+ : rows;
+ const [from, to] = call.range ?? [0, MAX_ROWS - 1];
+ return { data: sorted.slice(from, Math.min(to + 1, from + MAX_ROWS)), error: null };
+ };
+ const query = {
+ order(key) {
+ call.order = key;
+ return query;
+ },
+ range(from, to) {
+ call.range = [from, to];
+ return query;
+ },
+ then: (resolve, reject) => Promise.resolve(respond()).then(resolve, reject),
+ };
+ return query;
+ });
+ return calls;
+}
+
+const from = new Date('2026-10-02T00:00:00.000Z');
+const to = new Date('2026-10-08T23:59:59.999Z');
+
+afterEach(() => {
+ vi.useRealTimers();
+ supabaseAdmin.rpc.mockReset();
+});
+
+describe('promptScores', () => {
+ /**
+ * The regression: one response stops at 1000 rows without an error, so a
+ * brand with 2500 prompts had 1500 of them silently missing from the
+ * pulse's movers.
+ */
+ it('scores every prompt of a brand past 1000 prompts', async () => {
+ const calls = serve(Array.from({ length: 2500 }, (_, i) => summary(2499 - i)));
+
+ const scores = await promptScores('brand-1', from, to);
+
+ expect(scores.size).toBe(2500);
+ expect(scores.has(uuid(0))).toBe(true);
+ expect(scores.has(uuid(2499))).toBe(true);
+ expect(calls.map((c) => c.range)).toEqual([
+ [0, 999],
+ [1000, 1999],
+ [2000, 2999],
+ ]);
+ });
+
+ /** `range` without an order is a promise the database never made. */
+ it('pages the same window in prompt order', async () => {
+ const calls = serve(Array.from({ length: 1200 }, (_, i) => summary(i)));
+
+ await promptScores('brand-1', from, to);
+
+ for (const call of calls) {
+ expect(call).toMatchObject({
+ name: 'prompt_visibility_summaries',
+ params: {
+ p_brand_id: 'brand-1',
+ p_date_from: from.toISOString(),
+ p_date_to: to.toISOString(),
+ },
+ order: 'prompt_id',
+ });
+ }
+ });
+
+ it('takes a single request for a brand under 1000 prompts', async () => {
+ const calls = serve(Array.from({ length: 40 }, (_, i) => summary(i)));
+
+ expect((await promptScores('brand-1', from, to)).size).toBe(40);
+ expect(calls).toHaveLength(1);
+ });
+
+ /**
+ * The pulse's reads have retried a timeout since #687; paging must not lose
+ * that. The retry starts over from the first page, so no page is skipped.
+ */
+ it('retries a timed-out page from the first page', async () => {
+ vi.useFakeTimers();
+ const calls = serve(
+ Array.from({ length: 1500 }, (_, i) => summary(i)),
+ { failFirst: { message: 'canceling statement due to statement timeout' } },
+ );
+
+ const pending = promptScores('brand-1', from, to);
+ await vi.advanceTimersByTimeAsync(3_000);
+
+ expect((await pending).size).toBe(1500);
+ expect(calls.map((c) => c.range)).toEqual([
+ [0, 999],
+ [0, 999],
+ [1000, 1999],
+ ]);
+ });
+});
diff --git a/server/src/lib/rpc-pages.js b/server/src/lib/rpc-pages.js
new file mode 100644
index 000000000..a76d4cd3e
--- /dev/null
+++ b/server/src/lib/rpc-pages.js
@@ -0,0 +1,51 @@
+/**
+ * Reading every row of a set-returning RPC.
+ *
+ * PostgREST returns at most 1000 rows per request (`max_rows` in
+ * supabase/config.toml), and an RPC is no exception: the rest are dropped
+ * without an error. On 2026-10-16 that cost the Tracked Prompts page most of
+ * its Positions at 30 and 90 days; the server's per-prompt reads had the same
+ * cap, so a brand past 1000 prompts lost the rest from the pulse's movers and
+ * the uncited-mention signal.
+ *
+ * The web reads the same RPCs through web/src/lib/supabase/rpc-pages.ts; this
+ * is the server's copy of that loop.
+ */
+
+import { logger } from './logger.js';
+
+/** PostgREST's max_rows. A page larger than it would read short and end early. */
+export const RPC_PAGE_SIZE = 1000;
+
+/** 100,000 rows: far past any brand's window, and a bound on a runaway loop. */
+export const MAX_RPC_PAGES = 100;
+
+/**
+ * `page(from, to)` asks for one slice in a fixed order — each caller orders by
+ * its RPC's key, because `range` without an order is a promise the database
+ * never made — and slices are read until one comes back short. A result under
+ * 1000 rows still takes a single request.
+ *
+ * Each slice is its own query, so a row written in between can shift a later
+ * slice by one and repeat a row at the boundary: callers that count rows drop
+ * repeats by key.
+ *
+ * Returns supabase-js's own `{ data, error }` shape so call sites keep the
+ * error handling they already have. A failing slice is reported and what was
+ * read before it is discarded: a partial read would look like a complete one.
+ */
+export async function readAllRpcRows(name, page) {
+ const rows = [];
+ for (let i = 0; i < MAX_RPC_PAGES; i++) {
+ const from = i * RPC_PAGE_SIZE;
+ const { data, error } = await page(from, from + RPC_PAGE_SIZE - 1);
+ if (error) return { data: null, error };
+ rows.push(...(data ?? []));
+ if ((data ?? []).length < RPC_PAGE_SIZE) return { data: rows, error: null };
+ }
+ logger.warn(
+ { rpc: name, read: rows.length },
+ '[rpc-pages] page limit reached; the result covers only what was read',
+ );
+ return { data: rows, error: null };
+}
diff --git a/server/src/lib/rpc-pages.test.js b/server/src/lib/rpc-pages.test.js
new file mode 100644
index 000000000..14dc3cfa8
--- /dev/null
+++ b/server/src/lib/rpc-pages.test.js
@@ -0,0 +1,101 @@
+import { readFileSync } from 'node:fs';
+import { describe, expect, it, vi } from 'vitest';
+
+vi.mock('./logger.js', () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+import { logger } from './logger.js';
+import { MAX_RPC_PAGES, RPC_PAGE_SIZE, readAllRpcRows } from './rpc-pages.js';
+
+/** A set-returning RPC whose result has `total` rows, served as PostgREST would. */
+function rpcOf(total) {
+ const rows = Array.from({ length: total }, (_, i) => ({ n: i }));
+ return vi.fn(async (from, to) => ({
+ data: rows.slice(from, Math.min(to + 1, from + RPC_PAGE_SIZE)),
+ error: null,
+ }));
+}
+
+describe('readAllRpcRows', () => {
+ it('reads every row past the 1000-row cap, a page at a time', async () => {
+ const page = rpcOf(2500);
+
+ const { data, error } = await readAllRpcRows('summaries', page);
+
+ expect(error).toBeNull();
+ expect(data).toHaveLength(2500);
+ expect(data.at(-1)).toEqual({ n: 2499 });
+ expect(page.mock.calls).toEqual([
+ [0, 999],
+ [1000, 1999],
+ [2000, 2999],
+ ]);
+ });
+
+ it('takes a single request when the result is under a page', async () => {
+ const page = rpcOf(709);
+
+ expect((await readAllRpcRows('summaries', page)).data).toHaveLength(709);
+ expect(page).toHaveBeenCalledTimes(1);
+ });
+
+ it('asks once more when the last page is exactly full', async () => {
+ const page = rpcOf(1000);
+
+ expect((await readAllRpcRows('summaries', page)).data).toHaveLength(1000);
+ expect(page).toHaveBeenCalledTimes(2);
+ });
+
+ it('treats a page without data as the end', async () => {
+ const page = vi.fn(async () => ({ data: null, error: null }));
+
+ expect(await readAllRpcRows('summaries', page)).toEqual({ data: [], error: null });
+ expect(page).toHaveBeenCalledTimes(1);
+ });
+
+ /**
+ * A partial read would look like a complete one — the 1000-row cap all over
+ * again — so the first failing page wins and nothing read before it is kept.
+ */
+ it('reports the error of any page and discards what was read', async () => {
+ const timeout = { message: 'canceling statement due to statement timeout' };
+ const page = vi.fn(async (from) =>
+ from === 0
+ ? { data: Array.from({ length: RPC_PAGE_SIZE }, () => ({})), error: null }
+ : { data: null, error: timeout },
+ );
+
+ expect(await readAllRpcRows('summaries', page)).toEqual({ data: null, error: timeout });
+ expect(page).toHaveBeenCalledTimes(2);
+ });
+
+ it('stops at its page limit and says the result may be incomplete', async () => {
+ const full = Array.from({ length: RPC_PAGE_SIZE }, (_, n) => ({ n }));
+ const page = vi.fn(async () => ({ data: full, error: null }));
+
+ const { data } = await readAllRpcRows('summaries', page);
+
+ expect(data).toHaveLength(MAX_RPC_PAGES * RPC_PAGE_SIZE);
+ expect(page).toHaveBeenCalledTimes(MAX_RPC_PAGES);
+ expect(logger.warn).toHaveBeenCalledWith(
+ { rpc: 'summaries', read: MAX_RPC_PAGES * RPC_PAGE_SIZE },
+ expect.stringContaining('page limit reached'),
+ );
+ });
+});
+
+describe('RPC_PAGE_SIZE', () => {
+ /**
+ * A page larger than PostgREST's max_rows comes back short, and a short page
+ * reads as the last one: the loop would stop after the first response and
+ * bring the cap back without an error.
+ */
+ it('is no larger than the max_rows PostgREST is configured with', () => {
+ const config = readFileSync(new URL('../../../supabase/config.toml', import.meta.url), 'utf8');
+ const maxRows = Number(config.match(/^max_rows\s*=\s*(\d+)/m)?.[1]);
+
+ expect(maxRows).toBeGreaterThan(0);
+ expect(RPC_PAGE_SIZE).toBeLessThanOrEqual(maxRows);
+ });
+});
diff --git a/server/src/lib/signals/record.js b/server/src/lib/signals/record.js
index cfb21c2aa..1604ba6af 100644
--- a/server/src/lib/signals/record.js
+++ b/server/src/lib/signals/record.js
@@ -29,6 +29,7 @@
import supabaseAdmin from '../../config/supabase.js';
import { computePulseMetrics } from '../pulse/metrics.js';
import { logger } from '../logger.js';
+import { readAllRpcRows } from '../rpc-pages.js';
import { resolve } from '../../config/action-engine.js';
import { EXCLUSIVE_KIND_GROUPS, LIBRARY_KINDS } from './library/kinds.js';
import { libraryCandidates } from './library/detect.js';
@@ -227,20 +228,37 @@ function fromPulseEntry(entry) {
* the window's answers mention the brand at least once and cite it never —
* the brand is in the conversation but AI engines have nothing of its own
* to point at. One consolidated signal (brief §29), not one per prompt.
+ *
+ * One row per prompt, read a page at a time past PostgREST's 1000-row cap.
+ * Exported for the test that pins the paging behaviour.
*/
-async function uncitedMentionCandidates(brandId, now) {
+export async function uncitedMentionCandidates(brandId, now) {
const from = new Date(now.getTime() - detection.windowDays * DAY_MS);
- const { data, error } = await supabaseAdmin.rpc('prompt_visibility_summaries', {
- p_brand_id: brandId,
- p_date_from: from.toISOString(),
- p_date_to: now.toISOString(),
- });
+ const { data, error } = await readAllRpcRows('prompt_visibility_summaries', (start, end) =>
+ supabaseAdmin
+ .rpc('prompt_visibility_summaries', {
+ p_brand_id: brandId,
+ p_date_from: from.toISOString(),
+ p_date_to: now.toISOString(),
+ })
+ .order('prompt_id', { ascending: true })
+ .range(start, end),
+ );
if (error) throw new Error(error.message);
- const uncited = (data ?? []).filter(
- (row) =>
- Number(row.runs) > 0 && Number(row.total_mentions) > 0 && Number(row.total_citations) === 0,
- );
+ // By id, not by row: a prompt repeated at a page boundary is still one prompt.
+ const uncited = [
+ ...new Set(
+ (data ?? [])
+ .filter(
+ (row) =>
+ Number(row.runs) > 0 &&
+ Number(row.total_mentions) > 0 &&
+ Number(row.total_citations) === 0,
+ )
+ .map((row) => row.prompt_id),
+ ),
+ ];
if (uncited.length < detection.uncitedMinPrompts) return [];
return [
@@ -248,7 +266,7 @@ async function uncitedMentionCandidates(brandId, now) {
kind: 'uncited_mentions',
dedupKey: 'uncited_mentions',
source: ['ai_results'],
- payload: { promptIds: uncited.map((row) => row.prompt_id).slice(0, 50) },
+ payload: { promptIds: uncited.slice(0, 50) },
previousValue: null,
currentValue: uncited.length,
changeValue: null,
diff --git a/server/src/lib/signals/uncited-mentions.test.js b/server/src/lib/signals/uncited-mentions.test.js
new file mode 100644
index 000000000..7be5791d5
--- /dev/null
+++ b/server/src/lib/signals/uncited-mentions.test.js
@@ -0,0 +1,141 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+// record.js reaches the admin client at import time, whose config hard-exits
+// without SUPABASE_* env (as in CI). Its `rpc` is replaced per test below.
+vi.mock('../../config/supabase.js', () => ({ default: { rpc: vi.fn(), from: vi.fn() } }));
+vi.mock('../logger.js', () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+vi.mock('../pulse/metrics.js', () => ({ computePulseMetrics: vi.fn() }));
+
+import supabaseAdmin from '../../config/supabase.js';
+import { uncitedMentionCandidates } from './record.js';
+import { ENGINE_THRESHOLDS } from '../../config/action-engine.js';
+
+const { detection } = ENGINE_THRESHOLDS;
+const DAY_MS = 86_400_000;
+const MAX_ROWS = 1000;
+
+const uuid = (n) => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`;
+const uncited = (n) => ({ prompt_id: uuid(n), runs: 3, total_mentions: 2, total_citations: 0 });
+const cited = (n) => ({ prompt_id: uuid(n), runs: 3, total_mentions: 2, total_citations: 1 });
+
+/**
+ * PostgREST in miniature for one RPC: rows sorted by `.order()`, sliced by
+ * `.range()`, and never more than max_rows in one response. An un-ranged call
+ * gets the first 1000 rows and no sign that more exist — the defect itself.
+ * `afterFirstPage` runs once the first response is out, standing in for a
+ * write that lands between two pages.
+ */
+function serve(rows, { afterFirstPage = null } = {}) {
+ const calls = [];
+ supabaseAdmin.rpc.mockImplementation((name, params) => {
+ const call = { name, params, order: null, range: null };
+ calls.push(call);
+ const respond = () => {
+ const sorted = call.order
+ ? [...rows].sort((a, b) => a[call.order].localeCompare(b[call.order]))
+ : rows;
+ const [from, to] = call.range ?? [0, MAX_ROWS - 1];
+ const data = sorted.slice(from, Math.min(to + 1, from + MAX_ROWS));
+ if (calls.indexOf(call) === 0) afterFirstPage?.(rows);
+ return { data, error: null };
+ };
+ const query = {
+ order(key) {
+ call.order = key;
+ return query;
+ },
+ range(from, to) {
+ call.range = [from, to];
+ return query;
+ },
+ then: (resolve, reject) => Promise.resolve(respond()).then(resolve, reject),
+ };
+ return query;
+ });
+ return calls;
+}
+
+const now = new Date('2026-10-16T06:00:00.000Z');
+
+afterEach(() => {
+ supabaseAdmin.rpc.mockReset();
+});
+
+describe('uncitedMentionCandidates', () => {
+ /**
+ * The regression: one response stops at 1000 rows without an error, so a
+ * brand with 1500 uncited prompts was told it had at most 1000.
+ */
+ it('counts uncited prompts past the first 1000', async () => {
+ const rows = [
+ ...Array.from({ length: 1500 }, (_, i) => uncited(i)),
+ ...Array.from({ length: 300 }, (_, i) => cited(1500 + i)),
+ ];
+ const calls = serve(rows.reverse());
+
+ const [signal] = await uncitedMentionCandidates('brand-1', now);
+
+ expect(signal).toMatchObject({ kind: 'uncited_mentions', currentValue: 1500 });
+ expect(signal.payload.promptIds).toEqual(Array.from({ length: 50 }, (_, i) => uuid(i)));
+ expect(calls.map((c) => c.range)).toEqual([
+ [0, 999],
+ [1000, 1999],
+ ]);
+ });
+
+ /** `range` without an order is a promise the database never made. */
+ it('pages the detection window in prompt order', async () => {
+ const calls = serve(Array.from({ length: 1200 }, (_, i) => uncited(i)));
+
+ await uncitedMentionCandidates('brand-1', now);
+
+ const windowFrom = new Date(now.getTime() - detection.windowDays * DAY_MS);
+ for (const call of calls) {
+ expect(call).toMatchObject({
+ name: 'prompt_visibility_summaries',
+ params: {
+ p_brand_id: 'brand-1',
+ p_date_from: windowFrom.toISOString(),
+ p_date_to: now.toISOString(),
+ },
+ order: 'prompt_id',
+ });
+ }
+ });
+
+ /**
+ * A prompt answered for the first time between two pages sorts ahead of
+ * the rest and pushes the first page's last row onto the second. Counting
+ * rows would report one prompt more than the brand has.
+ */
+ it('counts a prompt repeated at a page boundary once', async () => {
+ const rows = Array.from({ length: 1500 }, (_, i) => uncited(i + 1));
+ serve(rows, { afterFirstPage: (served) => served.push(cited(0)) });
+
+ const [signal] = await uncitedMentionCandidates('brand-1', now);
+
+ expect(signal.currentValue).toBe(1500);
+ expect(new Set(signal.payload.promptIds).size).toBe(signal.payload.promptIds.length);
+ });
+
+ it('stays quiet below the minimum number of prompts', async () => {
+ serve(Array.from({ length: detection.uncitedMinPrompts - 1 }, (_, i) => uncited(i)));
+
+ expect(await uncitedMentionCandidates('brand-1', now)).toEqual([]);
+ });
+
+ it('throws when a page fails', async () => {
+ supabaseAdmin.rpc.mockImplementation(() => {
+ const query = {
+ order: () => query,
+ range: () => query,
+ then: (resolve) => resolve({ data: null, error: { message: 'statement timeout' } }),
+ };
+ return query;
+ });
+
+ await expect(uncitedMentionCandidates('brand-1', now)).rejects.toThrow('statement timeout');
+ });
+});