Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(volumes): split the prompt-id filter so large brands can load volumes (#805)
Gökhan Geyik93dd031cedbe
diff --git a/server/src/lib/chunked-in.js b/server/src/lib/chunked-in.jsnew file mode 100644index 000000000..957592340--- /dev/null+++ b/server/src/lib/chunked-in.js@@ -0,0 +1,64 @@+/**+ * Splitting a PostgREST `.in(column, ids)` filter across several requests.+ *+ * PostgREST carries `.in(...)` values in the query string, so one filter over+ * N uuids costs about 39 characters of URL each. The Prompts page asks for+ * every volume row a brand owns in a single call, and the largest brand's 402+ * prompts built a 15,792-character request — close enough to the edge's URL+ * ceiling that roughly one call in four was rejected before it ever reached+ * PostgREST. Those attempts left no trace in the database logs, the route+ * could only report that the query had failed, and the page dropped its+ * volume and competition columns with a "temporarily unavailable" toast.+ *+ * Every other brand's equivalent request is under 4 kB and has never failed,+ * so the cure is to keep each request in that range rather than to grow the+ * ceiling. Chunks run in parallel, so the extra round trips cost close to+ * nothing.+ *+ * `server/src/routes/prompts.js` already does this by hand for fan-out intent+ * lookups (`FANOUT_INTENT_LOOKUP_CHUNK`); this is the same idea, shared.+ */++/**+ * Ids per request. 100 uuids is roughly 4 kB of URL — the size the busiest+ * brands have always been served at.+ */+export const IN_FILTER_CHUNK = 100;++/** Split ids into `chunkSize`-sized slices, dropping duplicates. */+export function chunkIds(ids, chunkSize = IN_FILTER_CHUNK) {+ const unique = [...new Set(ids)];+ const chunks = [];+ for (let i = 0; i < unique.length; i += chunkSize) {+ chunks.push(unique.slice(i, i + chunkSize));+ }+ return chunks;+}++/**+ * Run one `.in(column, ids)` query as several smaller ones.+ *+ * `buildQuery` receives a slice of the ids and returns the PostgREST query for+ * that slice. Rows come back concatenated in chunk order, so a caller that+ * needs a global order or a global `limit` must apply it after the call.+ * Ordering *within* a single id's rows survives untouched, because an id only+ * ever appears in one chunk — which is what lets callers that group by id keep+ * their existing `.order(...)`.+ *+ * Returns supabase-js's own `{ data, error }` shape so call sites keep the+ * error handling they already have. The first failing chunk is reported and+ * the remaining results are discarded.+ */+export async function selectInChunks(ids, buildQuery, chunkSize = IN_FILTER_CHUNK) {+ const chunks = chunkIds(ids, chunkSize);+ if (chunks.length === 0) return { data: [], error: null };++ const settled = await Promise.all(chunks.map((chunk) => buildQuery(chunk)));++ const data = [];+ for (const result of settled) {+ if (result?.error) return { data: null, error: result.error };+ data.push(...(result?.data ?? []));+ }+ return { data, error: null };+}diff --git a/server/src/lib/chunked-in.test.js b/server/src/lib/chunked-in.test.jsnew file mode 100644index 000000000..e40214ecf--- /dev/null+++ b/server/src/lib/chunked-in.test.js@@ -0,0 +1,108 @@+import { describe, expect, it, vi } from 'vitest';+import { IN_FILTER_CHUNK, chunkIds, selectInChunks } from './chunked-in.js';++const uuid = (n) => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`;+const ids = (count) => Array.from({ length: count }, (_, i) => uuid(i));++/** What PostgREST puts on the wire for `.in(column, chunk)`. */+function inFilterLength(chunk) {+ return `prompt_id=in.%28${chunk.map(encodeURIComponent).join('%2C')}%29`.length;+}++describe('chunkIds', () => {+ it('splits at the chunk boundary', () => {+ expect(chunkIds(ids(250), 100).map((c) => c.length)).toEqual([100, 100, 50]);+ });++ it('returns no chunks for an empty list', () => {+ expect(chunkIds([])).toEqual([]);+ });++ it('keeps a list that already fits in one chunk', () => {+ expect(chunkIds(ids(4), 100)).toEqual([ids(4)]);+ });++ it('drops duplicates so a repeated id is not queried twice', () => {+ expect(chunkIds([uuid(1), uuid(2), uuid(1)], 100)).toEqual([[uuid(1), uuid(2)]]);+ });+});++describe('IN_FILTER_CHUNK', () => {+ /**+ * The regression this whole module exists for: 402 ids in one filter built a+ * 15,792-character request that the edge rejected about a quarter of the+ * time. A chunk has to stay far enough under that to be uncontroversial.+ */+ it('keeps one request well under the size that was being rejected', () => {+ expect(inFilterLength(ids(IN_FILTER_CHUNK))).toBeLessThan(8_000);+ });++ it('would have split the request that failed', () => {+ expect(chunkIds(ids(402)).length).toBeGreaterThan(1);+ });+});++describe('selectInChunks', () => {+ it('issues one query per chunk and concatenates the rows', async () => {+ const buildQuery = vi.fn(async (chunk) => ({+ data: chunk.map((id) => ({ id })),+ error: null,+ }));++ const { data, error } = await selectInChunks(ids(250), buildQuery, 100);++ expect(error).toBeNull();+ expect(buildQuery).toHaveBeenCalledTimes(3);+ expect(data).toHaveLength(250);+ expect(data[0]).toEqual({ id: uuid(0) });+ expect(data[249]).toEqual({ id: uuid(249) });+ });++ it('never queries for an empty id list', async () => {+ const buildQuery = vi.fn();++ const { data, error } = await selectInChunks([], buildQuery);++ expect(buildQuery).not.toHaveBeenCalled();+ expect(data).toEqual([]);+ expect(error).toBeNull();+ });++ it('reports the first failing chunk instead of returning partial rows', async () => {+ const buildQuery = vi.fn(async (chunk) =>+ chunk.includes(uuid(150))+ ? { data: null, error: { message: 'boom' } }+ : { data: chunk.map((id) => ({ id })), error: null },+ );++ const { data, error } = await selectInChunks(ids(250), buildQuery, 100);++ expect(data).toBeNull();+ expect(error).toEqual({ message: 'boom' });+ });++ it('keeps each id and the rows belonging to it in one chunk', async () => {+ // Callers group by id and rely on `.order(...)` holding within a group.+ // That only works because an id is never split across two requests.+ const seen = [];+ const buildQuery = async (chunk) => {+ seen.push(chunk);+ return {+ data: chunk.flatMap((id) => [+ { id, at: 'newer' },+ { id, at: 'older' },+ ]),+ error: null,+ };+ };++ const { data } = await selectInChunks(ids(250), buildQuery, 100);++ const chunksHolding = (id) => seen.filter((chunk) => chunk.includes(id)).length;+ expect(chunksHolding(uuid(99))).toBe(1);+ expect(chunksHolding(uuid(100))).toBe(1);++ const rowsFor = data.filter((row) => row.id === uuid(100));+ expect(rowsFor.map((row) => row.at)).toEqual(['newer', 'older']);+ });+});diff --git a/server/src/lib/opportunity-generator.js b/server/src/lib/opportunity-generator.jsindex 58899d4ac..81595391d 100644--- a/server/src/lib/opportunity-generator.js+++ b/server/src/lib/opportunity-generator.js@@ -8,6 +8,7 @@ import { z } from 'zod';import { resolveModel } from './ai-provider.js';import { getLanguageName } from './languages.js';import supabaseAdmin from '../config/supabase.js';+import { selectInChunks } from './chunked-in.js';import { logger } from './logger.js';import { OPPORTUNITIES_PER_RUN, OPPORTUNITY_COUNT_RULE } from './opportunity-limits.js';@@ -94,13 +95,21 @@ export async function generateContentOpportunities(brandId) {const promptIds = prompts.map((p) => p.id);+ // Both prompt-keyed filters are chunked: a large brand puts hundreds of+ // uuids in one query string, which is the request size the edge starts+ // rejecting (see lib/chunked-in.js). Grouping below still works, because a+ // prompt's rows all come from the same chunk and keep their order.const [volRes, resRes, compRes] = await Promise.all([- supabaseAdmin.from('prompt_volumes').select('*').in('prompt_id', promptIds),- supabaseAdmin- .from('prompt_results')- .select('prompt_id, visibility_score, competitor_mentions')- .in('prompt_id', promptIds)- .order('created_at', { ascending: false }),+ selectInChunks(promptIds, (chunk) =>+ supabaseAdmin.from('prompt_volumes').select('*').in('prompt_id', chunk),+ ),+ selectInChunks(promptIds, (chunk) =>+ supabaseAdmin+ .from('prompt_results')+ .select('prompt_id, visibility_score, competitor_mentions')+ .in('prompt_id', chunk)+ .order('created_at', { ascending: false }),+ ),supabaseAdmin.from('competitors').select('name').eq('brand_id', brandId),]);diff --git a/server/src/lib/pulse/metrics.js b/server/src/lib/pulse/metrics.jsindex cad1683fa..43f8adb31 100644--- a/server/src/lib/pulse/metrics.js+++ b/server/src/lib/pulse/metrics.js@@ -1,5 +1,6 @@import supabaseAdmin from '../../config/supabase.js';import { computeAiVisibilityScore } from '../../config/visibility-score.js';+import { chunkIds, selectInChunks } from '../chunked-in.js';import { logger } from '../logger.js';/**@@ -203,18 +204,26 @@ async function brandPrompts(brandId) {async function firstTimeCitations(promptById, from) {const promptIds = [...promptById.keys()];if (!promptIds.length) return [];- const { data } = await supabaseAdmin- .from('prompt_target_urls')- .select('url, label, prompt_id, first_cited_at')- .in('prompt_id', promptIds)- .gte('first_cited_at', from.toISOString())- .order('first_cited_at', { ascending: false })- .limit(5);- return (data ?? []).map((row) => ({- url: row.url,- label: row.label,- promptText: promptById.get(row.prompt_id)?.text ?? '',- }));+ const { data } = await selectInChunks(promptIds, (chunk) =>+ supabaseAdmin+ .from('prompt_target_urls')+ .select('url, label, prompt_id, first_cited_at')+ .in('prompt_id', chunk)+ .gte('first_cited_at', from.toISOString())+ .order('first_cited_at', { ascending: false })+ .limit(5),+ );+ // Each chunk brings back its own five, so the global five is taken here —+ // the `.limit(5)` above only narrows what each request has to carry.+ return (data ?? [])+ .slice()+ .sort((a, b) => String(b.first_cited_at).localeCompare(String(a.first_cited_at)))+ .slice(0, 5)+ .map((row) => ({+ url: row.url,+ label: row.label,+ promptText: promptById.get(row.prompt_id)?.text ?? '',+ }));}/** Platforms where the brand became visible for the first time ever. */@@ -251,30 +260,38 @@ async function newEngineAppearances(brandId, from) {async function lostCitationPrompts(brandId, promptById, now) {const promptIds = [...promptById.keys()];if (!promptIds.length) return [];- const { data: volumes } = await supabaseAdmin- .from('prompt_volumes')- .select('prompt_id')- .in('prompt_id', promptIds)- .gt('est_ai_volume', 0);+ const { data: volumes } = await selectInChunks(promptIds, (chunk) =>+ supabaseAdmin+ .from('prompt_volumes')+ .select('prompt_id')+ .in('prompt_id', chunk)+ .gt('est_ai_volume', 0),+ );const volumeIds = (volumes ?? []).map((v) => v.prompt_id);if (!volumeIds.length) return [];const since = new Date(now.getTime() - 14 * DAY_MS).toISOString();const PAGE = 1000;const rows = [];- for (let pageStart = 0; pageStart < 20000; pageStart += PAGE) {- const { data: page, error } = await supabaseAdmin- .from('prompt_results')- .select('prompt_id, citation_count, created_at')- .eq('brand_id', brandId)- .neq('platform', 'chatgpt-shopping')- .in('prompt_id', volumeIds)- .gte('created_at', since)- .order('created_at', { ascending: true })- .range(pageStart, pageStart + PAGE - 1);- if (error) throw new Error(`lost-citations scan failed: ${error.message}`);- rows.push(...(page ?? []));- if (!page || page.length < PAGE) break;+ // Paging runs inside each id chunk rather than across the whole list: the+ // id filter has to stay a manageable URL (see lib/chunked-in.js), and the+ // day-bucketing below is order-independent, so scanning brand by chunk+ // instead of strictly by date changes nothing downstream.+ for (const idChunk of chunkIds(volumeIds)) {+ for (let pageStart = 0; pageStart < 20000; pageStart += PAGE) {+ const { data: page, error } = await supabaseAdmin+ .from('prompt_results')+ .select('prompt_id, citation_count, created_at')+ .eq('brand_id', brandId)+ .neq('platform', 'chatgpt-shopping')+ .in('prompt_id', idChunk)+ .gte('created_at', since)+ .order('created_at', { ascending: true })+ .range(pageStart, pageStart + PAGE - 1);+ if (error) throw new Error(`lost-citations scan failed: ${error.message}`);+ rows.push(...(page ?? []));+ if (!page || page.length < PAGE) break;+ }}const byPrompt = new Map();diff --git a/server/src/routes/volumes.js b/server/src/routes/volumes.jsindex 6914fdee9..3eec9c47a 100644--- a/server/src/routes/volumes.js+++ b/server/src/routes/volumes.js@@ -8,6 +8,7 @@ import {PlanLimitError,} from '../lib/plan-guard.js';import supabaseAdmin from '../config/supabase.js';+import { selectInChunks } from '../lib/chunked-in.js';import { assertBrandAccess, assertPromptAccess } from '../lib/access.js';import { extractIntentKeywords } from '../lib/intent-extraction.js';import {@@ -421,6 +422,7 @@ router.get('/brand/:brandId', async (req, res) => {.eq('brand_id', brandId);if (psError) {+ req.log.error({ err: psError, brandId }, 'fetch prompt sets error');return res.status(500).json({error: 'Failed to fetch prompt sets',details: psError.message,@@ -439,6 +441,7 @@ router.get('/brand/:brandId', async (req, res) => {.in('prompt_set_id', setIds);if (pError) {+ req.log.error({ err: pError, brandId }, 'fetch prompts error');return res.status(500).json({ error: 'Failed to fetch prompts', details: pError.message });}@@ -448,16 +451,24 @@ router.get('/brand/:brandId', async (req, res) => {const promptIds = prompts.map((p) => p.id);- const { data: volumes, error: vError } = await supabaseAdmin- .from('prompt_volumes')- .select('*')- .in('prompt_id', promptIds)- .order('est_ai_volume', { ascending: false });+ // Chunked because this is the one filter that scales with the brand: the+ // largest brand's 402 prompts made a 15,792-character URL that the edge+ // rejected about a quarter of the time, and the page lost its volume and+ // competition columns whenever it did (see lib/chunked-in.js).+ const { data: volumes, error: vError } = await selectInChunks(promptIds, (chunk) =>+ supabaseAdmin.from('prompt_volumes').select('*').in('prompt_id', chunk),+ );if (vError) {+ req.log.error({ err: vError, brandId, promptCount: promptIds.length }, 'fetch volumes error');return res.status(500).json({ error: 'Failed to fetch volumes', details: vError.message });}+ // Ordering moved off the query now that rows arrive per chunk.+ // `est_ai_volume` is NOT NULL, so this reproduces the previous+ // `.order('est_ai_volume', { ascending: false })` exactly.+ volumes.sort((a, b) => b.est_ai_volume - a.est_ai_volume);+const promptMap = {};for (const p of prompts) {promptMap[p.id] = { text: p.text, category: p.category };diff --git a/server/src/workers/content-worker.js b/server/src/workers/content-worker.jsindex fa4b28537..96740652c 100644--- a/server/src/workers/content-worker.js+++ b/server/src/workers/content-worker.js@@ -8,6 +8,7 @@ import { z } from 'zod';import { resolveModel } from '../lib/ai-provider.js';import { getLanguageName } from '../lib/languages.js';import supabaseAdmin from '../config/supabase.js';+import { selectInChunks } from '../lib/chunked-in.js';import logger from '../lib/logger.js';import { OPPORTUNITIES_PER_RUN, OPPORTUNITY_COUNT_RULE } from '../lib/opportunity-limits.js';@@ -122,15 +123,23 @@ export async function processContentJob({ brandId, model, job }) {const promptIds = prompts.map((p) => p.id);+ // Chunked for the same reason as the volumes route: hundreds of prompt+ // uuids in one query string is the request size the edge starts rejecting+ // (see lib/chunked-in.js). A prompt's rows stay together in one chunk, so+ // the per-prompt ordering the maps below rely on is unchanged.const [volumeResult, resultResult, competitorResult] = await Promise.all([- supabaseAdmin.from('prompt_volumes').select('*').in('prompt_id', promptIds),- supabaseAdmin- .from('prompt_results')- .select(- 'prompt_id, visibility_score, mention_count, citation_count, sentiment, competitor_mentions',- )- .in('prompt_id', promptIds)- .order('created_at', { ascending: false }),+ selectInChunks(promptIds, (chunk) =>+ supabaseAdmin.from('prompt_volumes').select('*').in('prompt_id', chunk),+ ),+ selectInChunks(promptIds, (chunk) =>+ supabaseAdmin+ .from('prompt_results')+ .select(+ 'prompt_id, visibility_score, mention_count, citation_count, sentiment, competitor_mentions',+ )+ .in('prompt_id', chunk)+ .order('created_at', { ascending: false }),+ ),supabaseAdmin.from('competitors').select('id, name, domain').eq('brand_id', brandId),]);diff --git a/web/src/lib/actions/volumes.ts b/web/src/lib/actions/volumes.tsindex a104f153b..954eedfa6 100644--- a/web/src/lib/actions/volumes.ts+++ b/web/src/lib/actions/volumes.ts@@ -256,7 +256,12 @@ export async function getPromptVolumes(if (!res.ok) {const body = await res.json().catch(() => ({}));- throw new Error(body.error || `Server error: ${res.status}`);+ // The server sends the cause in `details`; dropping it is why a failing+ // volumes query reached the logs as a bare "Failed to fetch volumes" and+ // took a database-log reconstruction to identify.+ throw new Error(+ [body.error || `Server error: ${res.status}`, body.details].filter(Boolean).join(': '),+ );}const data = await res.json();