Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(tracking): stop discarding runs whose first result lands late (#710)

Gökhan Geyike502569de745
diff --git a/server/.env.example b/server/.env.example
index ae51e4caa..fbfd86d7f 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -119,8 +119,20 @@ AI_VOLUME_MULTIPLIER=0.15
# COMPOSIO_GA_MAX_PROPERTIES=500
# COMPOSIO_GA_STREAM_LOOKUPS=100
+# How long a run waits for Cloro's FIRST result before giving up (default 90
+# min). Nothing has come back yet at that point, so the stall and ghost exits
+# below — which compare successive pending counts — have nothing to measure.
+# On large brands the first callback has landed as late as 65 minutes in.
+# CLORO_FIRST_RESULT_WAIT_MIN=90
+
+# How long the tail may take, measured from the first result rather than from
+# submission (default 60 min), so a slow queue start doesn't eat the time the
+# rest of the delivery needs.
+# CLORO_DRAIN_TAIL_MIN=60
+
# Drain-stall tolerance: consecutive 15s polls with no new Cloro result before
-# a run stops waiting (default 100 ≈ 25 min; hard deadline is 60 min).
+# a run stops waiting (default 100 ≈ 25 min). Only counted after the first
+# result has arrived.
# CLORO_STALL_POLL_LIMIT=100
# Ghost-task tolerance: consecutive 15s polls with no new result before a run
diff --git a/server/src/lib/job-runner.js b/server/src/lib/job-runner.js
index cfe67af91..772d3bde8 100644
--- a/server/src/lib/job-runner.js
+++ b/server/src/lib/job-runner.js
@@ -83,10 +83,23 @@ export async function runTrackingJob(jobId, io) {
// Daily Pulse (#540): fire-and-forget after a full daily run — never
// for immediate/manual runs or single-prompt refreshes. The engine is
// self-contained: eligibility checks, dedup and error handling inside.
+ //
+ // Skipped when the run wasn't stamped (#702). An unstamped run left the
+ // 24h anchor on the previous run, so the pulse would recompute that same
+ // window and mail figures the recipient already received — which is
+ // exactly what happened for three days on the largest brand. The catch-up
+ // sweep picks the day back up once a run does stamp.
if (!immediate && !promptId && !promptIds?.length) {
- generatePulseForBrand(brandId).catch((err) => {
- logger.error({ err, brandId }, 'daily pulse trigger failed');
- });
+ if (result?.stamped) {
+ generatePulseForBrand(brandId).catch((err) => {
+ logger.error({ err, brandId }, 'daily pulse trigger failed');
+ });
+ } else {
+ logger.warn(
+ { brandId, resultCount: result?.resultCount },
+ 'skipping daily pulse — tracking run was not stamped, window would be stale',
+ );
+ }
}
} catch (err) {
if (abortController.signal.aborted) {
diff --git a/server/src/workers/tracking-worker.js b/server/src/workers/tracking-worker.js
index 39cdde239..28e5f6338 100644
--- a/server/src/workers/tracking-worker.js
+++ b/server/src/workers/tracking-worker.js
@@ -19,6 +19,30 @@ function resolveModelPlatform(model) {
return 'chatgpt';
}
+/**
+ * Which drain time budget, if any, has run out (#702).
+ *
+ * Two budgets rather than one cap measured from submission. Before anything
+ * has come back there is nothing to reason about — the stall and ghost exits
+ * both compare successive pending counts — so that phase gets its own, longer
+ * allowance. Once delivery starts, the tail is measured from the first result,
+ * so a slow queue start no longer consumes the time the tail needs.
+ *
+ * Returns 'no_first_result', 'tail_deadline', or null while within budget.
+ */
+export function drainBudgetExceeded({
+ now,
+ drainStartedAt,
+ firstResultAt,
+ firstResultWaitMs,
+ drainTailMs,
+}) {
+ if (firstResultAt === null || firstResultAt === undefined) {
+ return now - drainStartedAt >= firstResultWaitMs ? 'no_first_result' : null;
+ }
+ return now - firstResultAt >= drainTailMs ? 'tail_deadline' : null;
+}
+
/**
* True when every still-pending task was submitted longer ago than `maxAgeMs`.
*
@@ -304,15 +328,34 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,
const expectedSubmitted = submittedTaskIds.size;
if (expectedSubmitted > 0) {
- // Hard cap so a stuck Cloro queue can't keep a worker alive forever.
- const drainDeadline = Date.now() + 60 * 60 * 1000;
const drainPollMs = 15_000;
+ // Two separate budgets, because "nothing has come back yet" and "the
+ // tail is taking a while" are different situations (#702).
+ //
+ // A single 60-minute cap measured from submission discarded healthy
+ // runs: on the largest brand the first callback landed 62-65 minutes
+ // after submission three nights running, so the worker gave up two to
+ // five minutes before the delivery it was waiting for. The run then
+ // produced zero results at stamp time, the ledger row was deleted, and
+ // the dashboard and pulse stayed anchored to the previous day even
+ // though ~1800 results landed minutes later.
+ //
+ // Before the first result there is nothing to measure: the stall and
+ // ghost exits below both reason about *changes* in the pending set, so
+ // they are meaningless until at least one task has come back. That
+ // phase gets its own generous budget.
+ const firstResultWaitMs = (Number(process.env.CLORO_FIRST_RESULT_WAIT_MIN) || 90) * 60_000;
+ // Once delivery has started, this bounds the tail. Measured from the
+ // first result rather than from submission, so a slow queue start no
+ // longer eats the time the tail needs.
+ const drainTailMs = (Number(process.env.CLORO_DRAIN_TAIL_MIN) || 60) * 60_000;
// Give up early if delivery stalls — no new result for this many
// consecutive polls. Cloro delivers in bursts with quiet gaps: a real
// run went silent for 10+ minutes after the first ~600 results and
// then delivered the remaining ~1000, so the old ~10-min limit cut a
- // healthy run in half. ~25 min tolerates those gaps while the 60-min
- // drain deadline stays the hard cap on a truly stuck queue.
+ // healthy run in half. ~25 min tolerates those gaps while the tail
+ // budget above stays the hard cap on a queue that dies mid-delivery.
+ // Only counted once the first result has arrived.
const stallPollLimit = Number(process.env.CLORO_STALL_POLL_LIMIT) || 100;
// Ghost tasks — accepted by Cloro, never called back (google-aio does
// this whenever a query has no AI Overview) — used to burn the whole
@@ -326,8 +369,25 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,
let lastPending = expectedSubmitted;
let stalledPolls = 0;
+ const drainStartedAt = Date.now();
+ let firstResultAt = null;
+ let exitReason = 'drained';
+
+ for (;;) {
+ // Budget first, so it also bounds a poll that keeps failing — the
+ // retry path below skips the rest of the iteration.
+ const budgetExit = drainBudgetExceeded({
+ now: Date.now(),
+ drainStartedAt,
+ firstResultAt,
+ firstResultWaitMs,
+ drainTailMs,
+ });
+ if (budgetExit) {
+ exitReason = budgetExit;
+ break;
+ }
- while (Date.now() < drainDeadline) {
// Brand-scoped read (a handful of rows at most), intersected in memory
// with our own task_ids — avoids a giant `.in(...)` URL.
const { data: rows, error: drainErr } = await supabaseAdmin
@@ -349,6 +409,14 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,
const processed = expectedSubmitted - pending;
const allPendingAreOld = allTasksAreStale(ourRows, ghostTaskAgeMs);
+ if (processed > 0 && firstResultAt === null) {
+ firstResultAt = Date.now();
+ logger.info(
+ { brandId, waitedMs: firstResultAt - drainStartedAt, expected: expectedSubmitted },
+ 'cloro delivery started',
+ );
+ }
+
if (job) {
job.progress({
current: completedTasks + processed,
@@ -364,25 +432,43 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,
if (pending === 0) break;
+ // Stall and ghost detection reason about changes in the pending set,
+ // so they only mean anything once something has come back.
+ if (firstResultAt === null) {
+ await new Promise((r) => setTimeout(r, drainPollMs));
+ continue;
+ }
+
if (pending < lastPending) {
lastPending = pending;
stalledPolls = 0;
} else if (allPendingAreOld && stalledPolls + 1 >= ghostStallPolls) {
- logger.warn(
- { brandId, pending, expected: expectedSubmitted, stalledPolls: stalledPolls + 1 },
- 'cloro tasks are ghosts — delivery quiet and every remaining task is stale; continuing',
- );
+ stalledPolls += 1;
+ exitReason = 'ghosts';
break;
} else if (++stalledPolls >= stallPollLimit) {
- logger.warn(
- { brandId, pending, expected: expectedSubmitted },
- 'cloro delivery stalled — some tasks never returned; continuing',
- );
+ exitReason = 'stalled';
break;
}
await new Promise((r) => setTimeout(r, drainPollMs));
}
+
+ // Always log how the drain ended. Reconstructing this from the database
+ // the morning after — which is how #702 was diagnosed — is guesswork,
+ // because the timed-out path used to fall out of the loop silently.
+ const log = exitReason === 'drained' ? logger.info : logger.warn;
+ log.call(
+ logger,
+ {
+ brandId,
+ exitReason,
+ expected: expectedSubmitted,
+ elapsedMs: Date.now() - drainStartedAt,
+ firstResultAfterMs: firstResultAt ? firstResultAt - drainStartedAt : null,
+ },
+ `cloro drain finished (${exitReason})`,
+ );
}
completedTasks += expectedSubmitted;
@@ -568,6 +654,8 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,
// onto an empty window. On a count error we leave the row uncompleted —
// the dashboard keeps the previous completed run and the next run clears
// the dangling row.
+ let runStamped = false;
+
if (trackingRunId) {
const { count: runResultCount, error: countErr } = await supabaseAdmin
.from('prompt_results')
@@ -597,6 +685,7 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,
if (stampErr) {
logger.error({ err: stampErr, brandId, trackingRunId }, 'failed to stamp tracking run');
} else {
+ runStamped = true;
logger.info({ brandId, trackingRunId, runResultCount }, 'tracking run stamped');
}
} else if ((runResultCount ?? 0) > 0) {
@@ -611,5 +700,8 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,
}
}
- return { resultCount: insertedCount };
+ // `stamped` gates the Daily Pulse (#702): a run whose ledger row was
+ // refused never moved the 24h anchor, so its pulse would recompute the
+ // previous run's window and mail numbers the recipient already has.
+ return { resultCount: insertedCount, stamped: runStamped };
}
diff --git a/server/src/workers/tracking-worker.test.js b/server/src/workers/tracking-worker.test.js
index d95a4689c..2b2adbba6 100644
--- a/server/src/workers/tracking-worker.test.js
+++ b/server/src/workers/tracking-worker.test.js
@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest';
// SUPABASE_* env is absent (as in CI) — stub it before the import chain.
vi.mock('../config/supabase.js', () => ({ default: {} }));
-import { allTasksAreStale } from './tracking-worker.js';
+import { allTasksAreStale, drainBudgetExceeded } from './tracking-worker.js';
/**
* Ghost detection for the Cloro drain loop (#687).
@@ -59,3 +59,67 @@ describe('allTasksAreStale', () => {
expect(allTasksAreStale(rows, 30 * MINUTE, NOW)).toBe(false);
});
});
+
+/**
+ * Drain time budgets (#702).
+ *
+ * A single 60-minute cap measured from submission discarded three consecutive
+ * nightly runs on the largest brand: Cloro's first callback landed 62-65
+ * minutes after submission, so the worker gave up minutes before the delivery
+ * it was waiting for, then reported zero results and had its ledger row
+ * deleted — freezing the dashboard and the pulse on the previous day while
+ * ~1800 results landed just afterwards.
+ *
+ * Splitting the budget in two is what fixes it: waiting for the first result
+ * is a different situation from waiting out the tail, and only the second one
+ * has anything to measure.
+ */
+describe('drainBudgetExceeded', () => {
+ const FIRST_RESULT_WAIT = 90 * MINUTE;
+ const TAIL = 60 * MINUTE;
+ const budget = (over) =>
+ drainBudgetExceeded({
+ firstResultWaitMs: FIRST_RESULT_WAIT,
+ drainTailMs: TAIL,
+ ...over,
+ });
+
+ it('keeps waiting past the old 60-minute cap when nothing has arrived yet', () => {
+ // The exact case that broke: first delivery at 65 minutes.
+ expect(budget({ now: NOW + 65 * MINUTE, drainStartedAt: NOW, firstResultAt: null })).toBeNull();
+ });
+
+ it('gives up once the first-result budget is spent', () => {
+ expect(budget({ now: NOW + 90 * MINUTE, drainStartedAt: NOW, firstResultAt: null })).toBe(
+ 'no_first_result',
+ );
+ });
+
+ it('measures the tail from the first result, not from submission', () => {
+ // 80 minutes into the drain, but delivery only started 30 minutes ago:
+ // the tail still has time even though the total exceeds the tail budget.
+ expect(
+ budget({
+ now: NOW + 80 * MINUTE,
+ drainStartedAt: NOW,
+ firstResultAt: NOW + 50 * MINUTE,
+ }),
+ ).toBeNull();
+ });
+
+ it('gives up once the tail budget is spent', () => {
+ expect(
+ budget({
+ now: NOW + 120 * MINUTE,
+ drainStartedAt: NOW,
+ firstResultAt: NOW + 60 * MINUTE,
+ }),
+ ).toBe('tail_deadline');
+ });
+
+ it('treats an undefined first result the same as null', () => {
+ expect(budget({ now: NOW + 95 * MINUTE, drainStartedAt: NOW, firstResultAt: undefined })).toBe(
+ 'no_first_result',
+ );
+ });
+});