Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(tracking): page the pending-task read so large runs aren't cut short (#716)
Gökhan Geyik7161ed761d97
diff --git a/server/.env.example b/server/.env.exampleindex fbfd86d7f..bb0d9d1ec 100644--- a/server/.env.example+++ b/server/.env.example@@ -136,12 +136,19 @@ AI_VOLUME_MULTIPLIER=0.15# CLORO_STALL_POLL_LIMIT=100# Ghost-task tolerance: consecutive 15s polls with no new result before a run-# stops waiting *when every remaining task is already older than 30 minutes*-# (default 60 ≈ 15 min). Tasks that old can no longer be in flight — Cloro-# accepted them and will never call back — so this exits the quiet tail of a-# run without shortening the stall tolerance for tasks still plausibly coming.+# stops waiting *when every remaining task is already older than the ghost age+# below* (default 60 ≈ 15 min). Tasks that old can no longer be in flight —+# Cloro accepted them and will never call back — so this exits the quiet tail+# of a run without shortening the stall tolerance for tasks still plausibly+# coming.# CLORO_GHOST_STALL_POLLS=60+# How old a still-pending task must be to count as a ghost (default 30 min).+# The safe value tracks Cloro's delivery latency, which moves: first results+# have arrived 34 minutes after submission on one brand and 29.7 on another.+# Raise this if healthy runs start ending with `cloro drain finished (ghosts)`.+# CLORO_GHOST_TASK_AGE_MIN=30+# How many brands may run tracking jobs at once (default 2). Tracking mostly# waits on scraper callbacks, so raising this shortens the nightly cycle;# watch memory before going far above 4 on a small instance.diff --git a/server/src/workers/tracking-worker.js b/server/src/workers/tracking-worker.jsindex 28e5f6338..8ecd5286d 100644--- a/server/src/workers/tracking-worker.js+++ b/server/src/workers/tracking-worker.js@@ -19,6 +19,29 @@ function resolveModelPlatform(model) {return 'chatgpt';}+/**+ * PostgREST silently caps an un-paginated select at 1000 rows, so reading a+ * brand's pending tasks in one request under-reports any run that submitted+ * more than that (#714). Page through instead.+ *+ * Exported for the test that pins the paging behaviour; `fetchPage(offset)`+ * resolves to `{ data, error }` exactly as a PostgREST range query does.+ */+export const PENDING_PAGE_SIZE = 1000;++export async function fetchAllPendingRows(fetchPage, pageSize = PENDING_PAGE_SIZE) {+ const rows = [];+ for (let offset = 0; ; offset += pageSize) {+ const { data, error } = await fetchPage(offset);+ // A partial read is worse than no read: it looks like progress that never+ // happened. Surface the error and let the caller retry the whole poll.+ if (error) return { rows: null, error };+ const page = data ?? [];+ rows.push(...page);+ if (page.length < pageSize) return { rows, error: null };+ }+}+/*** Which drain time budget, if any, has run out (#702).*@@ -365,11 +388,17 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,// quiet gap alone is normal mid-burst, and old tasks alone are fine as// long as results are still landing.const ghostStallPolls = Number(process.env.CLORO_GHOST_STALL_POLLS) || 60;- const ghostTaskAgeMs = 30 * 60 * 1000;+ // How old a still-pending task has to be before it counts as a ghost.+ // Configurable because the safe value tracks Cloro's delivery latency,+ // which moves: one brand's first result has arrived 34 minutes after+ // submission, and another's at 29.7 — seconds under this threshold.+ // Raise it if healthy runs start exiting as 'ghosts'.+ const ghostTaskAgeMs = (Number(process.env.CLORO_GHOST_TASK_AGE_MIN) || 30) * 60_000;let lastPending = expectedSubmitted;let stalledPolls = 0;const drainStartedAt = Date.now();+ const drainStartedIso = new Date(drainStartedAt).toISOString();let firstResultAt = null;let exitReason = 'drained';@@ -388,12 +417,19 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,break;}- // Brand-scoped read (a handful of rows at most), intersected in memory- // with our own task_ids — avoids a giant `.in(...)` URL.- const { data: rows, error: drainErr } = await supabaseAdmin- .from('cloro_pending_tasks')- .select('task_id, submitted_at')- .eq('brand_id', brandId);+ // Brand-scoped read, intersected in memory with our own task_ids —+ // avoids a giant `.in(...)` URL. Paged, because PostgREST caps an+ // un-paginated select at 1000 rows: a run that submitted more than+ // that saw a pending count frozen at 1000, which read as "1130 tasks+ // already finished" on the first poll and as "nothing is moving" on+ // every poll after it (#714).+ const { rows, error: drainErr } = await fetchAllPendingRows((offset) =>+ supabaseAdmin+ .from('cloro_pending_tasks')+ .select('task_id, submitted_at')+ .eq('brand_id', brandId)+ .range(offset, offset + PENDING_PAGE_SIZE - 1),+ );// A transient read failure must NOT be read as "0 pending" — that would// break the loop early and report the run as finished while tasks are@@ -409,12 +445,27 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,const processed = expectedSubmitted - pending;const allPendingAreOld = allTasksAreStale(ourRows, ghostTaskAgeMs);+ // "Delivery started" has to mean a result actually landed, not just+ // that a pending row went away. The callback handler also deletes+ // rows for FAILED tasks, for a COMPLETED task with no response body,+ // and when the brand lookup fails — none of which produce a result.+ // A shrinking pending set is therefore not proof that anything+ // arrived, and treating it as proof starts the stall clock against a+ // run that has received nothing (#714).if (processed > 0 && firstResultAt === null) {- firstResultAt = Date.now();- logger.info(- { brandId, waitedMs: firstResultAt - drainStartedAt, expected: expectedSubmitted },- 'cloro delivery started',- );+ const { data: firstRows, error: firstErr } = await supabaseAdmin+ .from('prompt_results')+ .select('id')+ .eq('brand_id', brandId)+ .gte('created_at', drainStartedIso)+ .limit(1);+ if (!firstErr && (firstRows ?? []).length > 0) {+ firstResultAt = Date.now();+ logger.info(+ { brandId, waitedMs: firstResultAt - drainStartedAt, expected: expectedSubmitted },+ 'cloro delivery started',+ );+ }}if (job) {diff --git a/server/src/workers/tracking-worker.test.js b/server/src/workers/tracking-worker.test.jsindex 2b2adbba6..a92a24b60 100644--- a/server/src/workers/tracking-worker.test.js+++ b/server/src/workers/tracking-worker.test.js@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest';// SUPABASE_* env is absent (as in CI) — stub it before the import chain.vi.mock('../config/supabase.js', () => ({ default: {} }));-import { allTasksAreStale, drainBudgetExceeded } from './tracking-worker.js';+import { allTasksAreStale, drainBudgetExceeded, fetchAllPendingRows } from './tracking-worker.js';/*** Ghost detection for the Cloro drain loop (#687).@@ -123,3 +123,84 @@ describe('drainBudgetExceeded', () => {);});});++/**+ * Paged pending-task reads (#714).+ *+ * The drain read was un-paginated, and PostgREST silently caps those at 1000+ * rows. The largest brand submits ~2130 tasks, so every poll reported exactly+ * 1000 pending: on the first poll that read as "1130 tasks already finished"+ * — starting the stall clock against a run that had received nothing — and on+ * every poll after it as "nothing is moving", because the number never+ * changed. Fifteen minutes of that plus tasks crossing the ghost age exited+ * the run four seconds after the threshold, four minutes before its first+ * result arrived. Three nights of the brand's dashboard froze on it.+ *+ * It only ever bit the one brand over 1000 tasks, which is why it looked like+ * a brand-specific problem rather than a query bug.+ */+describe('fetchAllPendingRows', () => {+ const pageOf = (n, from = 0) =>+ Array.from({ length: n }, (_, i) => ({ task_id: `t${from + i}`, submitted_at: null }));++ it('returns every row when the set spans several pages', async () => {+ // 2130 rows: what the largest brand actually submits.+ const all = pageOf(2130);+ const fetchPage = async (offset) => ({+ data: all.slice(offset, offset + 1000),+ error: null,+ });++ const { rows, error } = await fetchAllPendingRows(fetchPage);++ expect(error).toBeNull();+ expect(rows).toHaveLength(2130);+ });++ it('stops after one request when the set fits in a page', async () => {+ const calls = [];+ const fetchPage = async (offset) => {+ calls.push(offset);+ return { data: pageOf(840), error: null };+ };++ const { rows } = await fetchAllPendingRows(fetchPage);++ expect(rows).toHaveLength(840);+ expect(calls).toEqual([0]);+ });++ it('asks for one more page when a page comes back exactly full', async () => {+ const calls = [];+ const fetchPage = async (offset) => {+ calls.push(offset);+ return { data: offset === 0 ? pageOf(1000) : [], error: null };+ };++ const { rows } = await fetchAllPendingRows(fetchPage);++ expect(rows).toHaveLength(1000);+ expect(calls).toEqual([0, 1000]);+ });++ it('reports an error rather than returning a partial set', async () => {+ // A half-read pending set looks like progress that never happened, which+ // is precisely the failure this whole change exists to prevent.+ const fetchPage = async (offset) =>+ offset === 0+ ? { data: pageOf(1000), error: null }+ : { data: null, error: { message: 'boom' } };++ const { rows, error } = await fetchAllPendingRows(fetchPage);++ expect(rows).toBeNull();+ expect(error).toEqual({ message: 'boom' });+ });++ it('handles an empty pending set', async () => {+ const { rows, error } = await fetchAllPendingRows(async () => ({ data: [], error: null }));++ expect(rows).toEqual([]);+ expect(error).toBeNull();+ });+});