Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
perf(tracking): start the first run after a payment at once, and fill its pages as answers arrive (#1773)
Cihan5f5453bbfd86
diff --git a/server/.env.example b/server/.env.exampleindex 6ce519c92..ef818c0cf 100644--- a/server/.env.example+++ b/server/.env.example@@ -272,3 +272,8 @@ AI_VOLUME_MULTIPLIER=0.45# waits on scraper callbacks, so raising this shortens the nightly cycle;# watch memory before going far above 4 on a small instance.# TRACKING_CONCURRENCY=2++# Slots the first run after a payment may take beyond TRACKING_CONCURRENCY+# (default 1), so it never waits behind the nightly runs. It goes ahead of the+# queue either way; 0 turns only the extra slot off.+# TRACKING_PRIORITY_SLOTS=1diff --git a/server/src/lib/first-run-rollups.js b/server/src/lib/first-run-rollups.jsnew file mode 100644index 000000000..83e11053f--- /dev/null+++ b/server/src/lib/first-run-rollups.js@@ -0,0 +1,118 @@+/**+ * A brand's first tracking run, folded into the daily rollups while its+ * answers are still landing.+ *+ * The rollups (Insights 00066, Citations 00118, Mentions 00157) take a run's+ * days once it stamps (run-settlement.js), so a half-delivered run never+ * replaces a complete day. A brand's first run has no complete day to keep:+ * until it stamps — at 95% of its answers, or when nothing is pending, or up+ * to two hours later — every view that reads the rollups (Citations,+ * Mentions, Brand Visibility's 7/30/90-day ranges) shows nothing. So a first+ * run refreshes its days as answers land, now and then, and the stamp's own+ * refresh settles them.+ *+ * The SQL refresh functions take a per-brand advisory lock, so two refreshes+ * of the same days never interleave. They are still started one at a time+ * here, and the stamp waits for the last (run-settlement.js): a refresh+ * started while another holds the lock only sits on a database connection+ * until it can go.+ */++import supabaseAdmin from '../config/supabase.js';+import { refreshRunDays } from './insights-rollups.js';+import { logger } from './logger.js';++/** At least this long between two refreshes of a first run's days. */+export const PROGRESSIVE_REFRESH_MS = 2 * 60_000;++/**+ * And at least this many times as long as the last refresh took, so a brand+ * whose refresh is slow does not refresh back to back.+ */+export const REFRESH_DURATION_FACTOR = 5;++/**+ * How often a first run that ended short of complete checks again whether it+ * can stamp (run-settlement.js; two counts a poll). Its stamp is what fills+ * the Action Center, so a first run checks every 30 s instead of every 2 min.+ */+export const FIRST_RUN_LATE_POLL_MS = 30_000;++/**+ * Whether a brand has never had a run stamped, so the run starting now is its+ * first. A failed read says no: the run then behaves as any other.+ */+export async function isFirstRun(brandId) {+ const { data, error } = await supabaseAdmin+ .from('tracking_runs')+ .select('id')+ .eq('brand_id', brandId)+ .not('completed_at', 'is', null)+ .limit(1);+ if (error) {+ logger.warn({ err: error, brandId }, 'could not tell whether this is the first run');+ return false;+ }+ return (data ?? []).length === 0;+}++/**+ * The refresher for one run. All three do nothing unless `enabled`.+ *+ * - `note(landed)`, with a count that changes as answers land: starts a+ * refresh when it changed since the last one, none is running, and the+ * wait since the last one started is over. Never waits.+ * - `idle()`: resolves once no refresh is running. The stamp waits for it.+ * - `flush(landed)`: once idle, refreshes now, whatever the wait; for a run+ * whose stamp has to wait for late answers.+ *+ * @param {{ brandId: string, startedAt: string | null, enabled: boolean,+ * intervalMs?: number, now?: () => number }} opts+ */+export function progressiveRollups({+ brandId,+ startedAt,+ enabled,+ intervalMs = PROGRESSIVE_REFRESH_MS,+ now = Date.now,+}) {+ const on = Boolean(enabled && startedAt);+ let lastStartedAt = null;+ let lastTookMs = 0;+ let lastLanded = 0;+ let inFlight = null;++ const start = (landed) => {+ const began = now();+ lastStartedAt = began;+ lastLanded = landed;+ inFlight = refreshRunDays(brandId, startedAt)+ .catch((err) => {+ logger.warn({ err, brandId }, 'first-run rollup refresh failed');+ })+ .finally(() => {+ lastTookMs = now() - began;+ inFlight = null;+ });+ return inFlight;+ };++ const idle = async () => {+ while (inFlight) await inFlight;+ };++ return {+ note(landed) {+ if (!on || inFlight || landed === lastLanded) return;+ const wait = Math.max(intervalMs, REFRESH_DURATION_FACTOR * lastTookMs);+ if (lastStartedAt !== null && now() - lastStartedAt < wait) return;+ start(landed);+ },+ idle,+ async flush(landed) {+ if (!on) return;+ await idle();+ await start(landed);+ },+ };+}diff --git a/server/src/lib/first-run-rollups.test.js b/server/src/lib/first-run-rollups.test.jsnew file mode 100644index 000000000..7bb6ddddc--- /dev/null+++ b/server/src/lib/first-run-rollups.test.js@@ -0,0 +1,208 @@+import { beforeEach, describe, expect, it, vi } from 'vitest';++/**+ * A brand's first run fills the rollups while its answers land, instead of+ * leaving Citations, Mentions and the wider Visibility ranges empty until it+ * stamps.+ */++const runs = vi.hoisted(() => ({ rows: [], error: null, filters: [] }));+vi.mock('../config/supabase.js', () => ({+ default: {+ from: () => {+ const chain = {+ select: () => chain,+ eq: (...args) => {+ runs.filters.push(['eq', ...args]);+ return chain;+ },+ not: (...args) => {+ runs.filters.push(['not', ...args]);+ return chain;+ },+ limit: () => Promise.resolve({ data: runs.error ? null : runs.rows, error: runs.error }),+ };+ return chain;+ },+ },+}));++// Each refresh stays running until released.+const refresh = vi.hoisted(() => ({ calls: [], releases: [] }));+vi.mock('./insights-rollups.js', () => ({+ refreshRunDays: vi.fn(+ (...args) =>+ new Promise((resolve) => {+ refresh.calls.push(args);+ refresh.releases.push(() => resolve(true));+ }),+ ),+}));+vi.mock('./logger.js', () => ({ logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn() } }));++const { isFirstRun, progressiveRollups, PROGRESSIVE_REFRESH_MS } =+ await import('./first-run-rollups.js');++const STARTED = '2026-10-29T10:00:00.000Z';+const MINUTE = 60_000;+let clock;+const make = (enabled = true) =>+ progressiveRollups({ brandId: 'b1', startedAt: STARTED, enabled, now: () => clock });+const tick = () => new Promise((resolve) => setTimeout(resolve, 0));+const releaseAll = async () => {+ refresh.releases.splice(0).forEach((release) => release());+ await tick();+};++beforeEach(() => {+ clock = 0;+ refresh.calls.length = 0;+ refresh.releases.length = 0;+ runs.rows = [];+ runs.error = null;+ runs.filters.length = 0;+});++describe('progressiveRollups', () => {+ it('refreshes the run days as soon as the first answers land', () => {+ const rollups = make();+ rollups.note(0);+ expect(refresh.calls).toEqual([]);++ rollups.note(3);+ expect(refresh.calls).toEqual([['b1', STARTED]]);+ });++ it('refreshes at most once per interval, and only when more have landed', async () => {+ const rollups = make();+ rollups.note(3);+ await releaseAll();++ clock += PROGRESSIVE_REFRESH_MS - 1;+ rollups.note(10);+ expect(refresh.calls).toHaveLength(1);++ clock += 1;+ rollups.note(10);+ expect(refresh.calls).toHaveLength(2);+ await releaseAll();++ // Nothing new since: no refresh, however long it has been.+ clock += 10 * PROGRESSIVE_REFRESH_MS;+ rollups.note(10);+ expect(refresh.calls).toHaveLength(2);+ });++ it('waits five times as long as a slow refresh took before the next', async () => {+ const rollups = make();+ rollups.note(3);+ // This brand's refresh takes a minute.+ clock += MINUTE;+ await releaseAll();++ // Two minutes since it started is not enough: five minutes are.+ clock += 3 * MINUTE;+ rollups.note(10);+ expect(refresh.calls).toHaveLength(1);++ clock += MINUTE;+ rollups.note(10);+ expect(refresh.calls).toHaveLength(2);+ });++ it('never starts a refresh while one is running', () => {+ const rollups = make();+ rollups.note(3);+ clock += 10 * PROGRESSIVE_REFRESH_MS;+ rollups.note(20);+ expect(refresh.calls).toHaveLength(1);+ });++ it('is idle only once the running refresh is done', async () => {+ const rollups = make();+ rollups.note(3);++ let idle = false;+ const waiting = rollups.idle().then(() => {+ idle = true;+ });+ await tick();+ expect(idle).toBe(false);++ await releaseAll();+ await waiting;+ expect(idle).toBe(true);+ // No refresh of its own.+ expect(refresh.calls).toHaveLength(1);+ });++ it('on flush, waits for the running refresh, then refreshes at once', async () => {+ const rollups = make();+ rollups.note(3);++ let flushed = false;+ const flushing = rollups.flush(3).then(() => {+ flushed = true;+ });+ await tick();+ expect(refresh.calls).toHaveLength(1);++ await releaseAll();+ // Within the interval and with the same count: refreshed all the same.+ expect(refresh.calls).toHaveLength(2);+ expect(flushed).toBe(false);++ await releaseAll();+ await flushing;+ expect(flushed).toBe(true);+ });++ it('follows a new count after a flush, as the late answers land', async () => {+ const rollups = make();+ const flushing = rollups.flush(40);+ await tick();+ await releaseAll();+ await flushing;++ clock += PROGRESSIVE_REFRESH_MS;+ rollups.note(40);+ expect(refresh.calls).toHaveLength(1);+ rollups.note(44);+ expect(refresh.calls).toHaveLength(2);+ });++ it('does nothing for a run that is not a first run', async () => {+ const rollups = make(false);+ rollups.note(5);+ await rollups.flush(9);+ await rollups.idle();+ expect(refresh.calls).toEqual([]);+ });++ it('does nothing without the run row to date it', async () => {+ const rollups = progressiveRollups({ brandId: 'b1', startedAt: null, enabled: true });+ rollups.note(5);+ await rollups.flush(9);+ expect(refresh.calls).toEqual([]);+ });+});++describe('isFirstRun', () => {+ it('is true for a brand with no stamped run', async () => {+ expect(await isFirstRun('b1')).toBe(true);+ expect(runs.filters).toEqual([+ ['eq', 'brand_id', 'b1'],+ ['not', 'completed_at', 'is', null],+ ]);+ });++ it('is false once a run has stamped', async () => {+ runs.rows = [{ id: 'run1' }];+ expect(await isFirstRun('b1')).toBe(false);+ });++ it('is false when the ledger cannot be read', async () => {+ runs.error = { message: 'timeout' };+ expect(await isFirstRun('b1')).toBe(false);+ });+});diff --git a/server/src/lib/insights-rollups.js b/server/src/lib/insights-rollups.jsindex 0e281d538..6f972cbd8 100644--- a/server/src/lib/insights-rollups.js+++ b/server/src/lib/insights-rollups.js@@ -6,7 +6,9 @@* them, by calling the `refresh_insights_daily` SQL function. Two callers:** - tracking-worker, the moment a run's ledger row is stamped — so a day- * enters the rollups only once its run completed, never mid-stream+ * enters the rollups only once its run completed, never mid-stream; a+ * brand's first run, which has no complete day to keep, also while its+ * answers land (first-run-rollups.js)* - the daily sweep, which re-refreshes the trailing days for every brand* as a backstop: runs that never stamped (partial, crashed, ghost tasks)* and out-of-run writes (a prompt analyzed the moment it was added, #754)@@ -55,17 +57,23 @@ export async function refreshInsightsDaily(brandId, dayFrom, dayTo) {}/**- * Refresh the days a completed tracking run touched: from the run's start- * day through today (a run crossing UTC midnight lands rows on two days).- * The Citations (00118) and Mentions (00157) rollups are refreshed for the- * same days.+ * Refresh the days a tracking run touched: from the run's start day through+ * today (a run crossing UTC midnight lands rows on two days). The Citations+ * (00118) and Mentions (00157) rollups are refreshed for the same days.+ * Resolves to whether the Insights refresh succeeded; never throws.*/-export async function refreshForCompletedRun(brandId, runStartedAtIso) {+export async function refreshRunDays(brandId, runStartedAtIso) {const dayFrom = utcDay(runStartedAtIso);const dayTo = utcDay();const ok = await refreshInsightsDaily(brandId, dayFrom, dayTo);await refreshCitationsDaily(brandId, dayFrom, dayTo);await refreshMentionsDaily(brandId, dayFrom, dayTo);+ return ok;+}++/** refreshRunDays for a run that has just stamped, then the Perception views. */+export async function refreshForCompletedRun(brandId, runStartedAtIso) {+ const ok = await refreshRunDays(brandId, runStartedAtIso);// The Perception tab's default view (00241) follows the rollups as they// now are. Not waited for: it never throws, and the run's stamp shouldn't// wait on three more reads.diff --git a/server/src/lib/job-manager.js b/server/src/lib/job-manager.jsindex 8cc45b366..eaccbbac3 100644--- a/server/src/lib/job-manager.js+++ b/server/src/lib/job-manager.js@@ -119,7 +119,8 @@ export async function listWaitingJobs({ idleMs, maxAgeMs }) {const now = Date.now();const { data, error } = await supabaseAdmin.from('jobs')- .select('id, type, brand_id')+ // `data` too: a run after a payment keeps its place ahead of the queue.+ .select('id, type, brand_id, data').eq('status', 'waiting').lt('updated_at', new Date(now - idleMs).toISOString()).gt('created_at', new Date(now - maxAgeMs).toISOString())diff --git a/server/src/lib/job-runner.js b/server/src/lib/job-runner.jsindex 60ecc0bb6..35e51573a 100644--- a/server/src/lib/job-runner.js+++ b/server/src/lib/job-runner.js@@ -30,6 +30,20 @@ import logger from './logger.js';// the cycle, watching memory on small instances.let activeTrackingCount = 0;const MAX_CONCURRENT_TRACKING = Number(process.env.TRACKING_CONCURRENCY) || 2;++// The first run after a payment is the person's first look at the product, and+// they are watching it. It goes ahead of the queue: it may take this many+// slots beyond the limit, so it never waits behind the nightly runs, each of+// which holds its slot through a Cloro drain that can last an hour; and while+// one still waits for a slot, the others step aside. 0 turns the extra slot+// off, leaving only the place in the queue.+const PRIORITY_EXTRA_SLOTS = /^\d+$/.test(process.env.TRACKING_PRIORITY_SLOTS ?? '')+ ? Number(process.env.TRACKING_PRIORITY_SLOTS)+ : 1;+const PRIORITY_RETRY_MS = 1000;+const REGULAR_RETRY_MS = 5000;+// Priority jobs waiting for a slot right now.+const priorityWaiting = new Set();let activeContentCount = 0;const MAX_CONCURRENT_CONTENT = 2;@@ -81,19 +95,26 @@ async function skipUnpaid(jobId, brandId) {/*** Run a tracking job. Call without await (fire-and-forget).- * Handles concurrency gating, retry, and Socket.IO emit.+ * Handles concurrency gating, retry, and Socket.IO emit. `priority` is for+ * the first run after a payment (see PRIORITY_EXTRA_SLOTS).*/-export function runTrackingJob(jobId, io) {+export function runTrackingJob(jobId, io, { priority = false } = {}) {if (queuedJobs.has(jobId)) return;queuedJobs.add(jobId);- void attemptTrackingJob(jobId, io);+ void attemptTrackingJob(jobId, io, priority);}-async function attemptTrackingJob(jobId, io) {- if (activeTrackingCount >= MAX_CONCURRENT_TRACKING) {- setTimeout(() => attemptTrackingJob(jobId, io), 5000);+async function attemptTrackingJob(jobId, io, priority = false) {+ const limit = MAX_CONCURRENT_TRACKING + (priority ? PRIORITY_EXTRA_SLOTS : 0);+ if (activeTrackingCount >= limit || (!priority && priorityWaiting.size > 0)) {+ if (priority) priorityWaiting.add(jobId);+ setTimeout(+ () => attemptTrackingJob(jobId, io, priority),+ priority ? PRIORITY_RETRY_MS : REGULAR_RETRY_MS,+ );return;}+ priorityWaiting.delete(jobId);// Take the slot before the first await. Otherwise every timer that fires// while the claim is in flight sees the same free slot, and a night once@@ -219,7 +240,7 @@ async function attemptTrackingJob(jobId, io) {.eq('id', jobId);retrying = true;- setTimeout(() => attemptTrackingJob(jobId, io), delay);+ setTimeout(() => attemptTrackingJob(jobId, io, priority), delay);} else {await failJob(jobId, err.message);}@@ -333,8 +354,11 @@ export async function recoverWaitingJobs(io) {if (await cancelWaitingJob(job.id, 'subscription_inactive')) cancelled++;continue;}- if (job.type === 'tracking') runTrackingJob(job.id, io);- else runContentJob(job.id, io);+ if (job.type === 'tracking') {+ runTrackingJob(job.id, io, { priority: job.data?.afterPayment === true });+ } else {+ runContentJob(job.id, io);+ }recovered++;}diff --git a/server/src/lib/job-runner.test.js b/server/src/lib/job-runner.test.jsindex 0545a3133..791c5a58b 100644--- a/server/src/lib/job-runner.test.js+++ b/server/src/lib/job-runner.test.js@@ -118,6 +118,108 @@ describe('runTrackingJob', () => {});});+/**+ * The first run after a payment goes ahead of the queue: someone is watching+ * it, and the nightly runs can hold every slot for an hour.+ */+describe('runTrackingJob, the first run after a payment', () => {+ it('starts at once in its extra slot while the nightly runs hold every slot', async () => {+ const releases = heldRuns();+ runTrackingJob('night-1');+ runTrackingJob('night-2');+ await flush();+ expect(processTrackingJob).toHaveBeenCalledTimes(MAX);++ runTrackingJob('first', undefined, { priority: true });+ await flush();+ expect(processTrackingJob).toHaveBeenCalledTimes(MAX + 1);+ expect(processTrackingJob.mock.calls[MAX][0].brandId).toBe('brand-first');++ // The extra slot is the first run's only: a regular job still waits.+ runTrackingJob('night-3');+ await vi.advanceTimersByTimeAsync(5000);+ expect(processTrackingJob).toHaveBeenCalledTimes(MAX + 1);++ await drain(releases);+ expect(processTrackingJob).toHaveBeenCalledTimes(MAX + 2);+ });++ it('takes the next free slot when even its extra one is taken, ahead of regular jobs', async () => {+ const releases = heldRuns();+ for (const id of ['night-1', 'night-2', 'night-3']) runTrackingJob(id);+ runTrackingJob('first-1', undefined, { priority: true });+ await flush();+ // night-3 waits, retrying every 5 s from now.+ expect(processTrackingJob).toHaveBeenCalledTimes(MAX + 1);++ // Another payment, every slot still taken.+ await vi.advanceTimersByTimeAsync(4500);+ runTrackingJob('first-2', undefined, { priority: true });+ await flush();++ // Two slots free up just before night-3 tries again: night-3 steps aside.+ releases[0]();+ releases[1]();+ await vi.advanceTimersByTimeAsync(500);+ const started = () => processTrackingJob.mock.calls.map(([job]) => job.brandId);+ expect(started()).not.toContain('brand-night-3');++ await vi.advanceTimersByTimeAsync(500);+ expect(started()).toContain('brand-first-2');+ expect(started()).not.toContain('brand-night-3');++ await drain(releases);+ expect(started()).toContain('brand-night-3');+ });++ it('keeps its extra slot when it is retried after a failure', async () => {+ const releases = heldRuns();+ const held = processTrackingJob.getMockImplementation();+ processTrackingJob.mockImplementation((job) =>+ job.brandId === 'brand-first' && processTrackingJob.mock.calls.length <= MAX + 1+ ? Promise.reject(new Error('cloro down'))+ : held(job),+ );+ manager.getJob.mockResolvedValue({ attempts: 1, max_attempts: 3 });+ runTrackingJob('night-1');+ runTrackingJob('night-2');+ await flush();+ runTrackingJob('first', undefined, { priority: true });+ await flush();+ expect(manager.failJob).toHaveBeenCalledWith('first', 'cloro down');++ // The retry waits attempts × 30 s; the nightly runs still hold both slots.+ await vi.advanceTimersByTimeAsync(30_000);+ const firstRuns = processTrackingJob.mock.calls.filter(+ ([job]) => job.brandId === 'brand-first',+ );+ expect(firstRuns).toHaveLength(2);++ await drain(releases);+ });++ it('keeps its place when the recovery sweep restarts it', async () => {+ const releases = heldRuns();+ runTrackingJob('night-1');+ runTrackingJob('night-2');+ await flush();++ manager.listWaitingJobs.mockResolvedValue([+ { id: 'lost-first', type: 'tracking', brand_id: 'b', data: { afterPayment: true } },+ { id: 'lost-night', type: 'tracking', brand_id: 'b', data: { immediate: false } },+ ]);+ await recoverWaitingJobs();+ await flush();++ expect(processTrackingJob.mock.calls.map(([job]) => job.brandId)).toEqual([+ 'brand-night-1',+ 'brand-night-2',+ 'brand-lost-first',+ ]);+ await drain(releases);+ });+});+/*** Once the plan has ended (2026-10-27): a job is cancelled when it is* claimed, however it was started, before anything is tracked or written.diff --git a/server/src/lib/run-settlement.js b/server/src/lib/run-settlement.jsindex 905ac5006..7482798d0 100644--- a/server/src/lib/run-settlement.js+++ b/server/src/lib/run-settlement.js@@ -111,7 +111,7 @@ async function act(decision, { brandId, runId, startedAt, plannedTasks }, resultreturn false;}-async function awaitLateResults(run, { pollMs, waitMs }) {+async function awaitLateResults(run, { pollMs, waitMs, rollups }) {const deadline = Date.now() + waitMs;for (;;) {await new Promise((r) => setTimeout(r, pollMs));@@ -131,11 +131,16 @@ async function awaitLateResults(run, { pollMs, waitMs }) {plannedTasks: run.plannedTasks,final: pastDeadline || pending === 0,});- if (decision === 'wait') continue;+ if (decision === 'wait') {+ // A first run's rollups follow the late answers too.+ rollups?.note(resultCount);+ continue;+ }logger.info({ brandId: run.brandId, runId: run.runId, resultCount, plannedTasks: run.plannedTasks },'late results settled the tracking run',);+ await rollups?.idle();return act(decision, run, resultCount);}}@@ -143,13 +148,18 @@ async function awaitLateResults(run, { pollMs, waitMs }) {/*** Settle a full run's ledger row at the end of its drain.*+ * `rollups` is a brand's first run's refresher (first-run-rollups.js): the+ * stamp waits for a refresh it still has running, as it refreshes the same+ * days; and a run that has to wait for late answers refreshes once now, then+ * as they land.+ ** @returns {Promise<{ stamped: boolean, lateStamp: Promise<boolean> | null }>}* `stamped` when the row stamped now; otherwise `lateStamp` resolves true* if and when it stamps later, false if it never does.*/export async function settleTrackingRun({ brandId, runId, startedAt, plannedTasks },- { pollMs = LATE_POLL_MS, waitMs = LATE_WAIT_MS } = {},+ { pollMs = LATE_POLL_MS, waitMs = LATE_WAIT_MS, rollups = null } = {},) {const run = { brandId, runId, startedAt, plannedTasks };const resultCount = await countRunResults(brandId, startedAt);@@ -162,14 +172,19 @@ export async function settleTrackingRun(const pending = await countRunPending(brandId, startedAt);const decision = settleDecision({ resultCount, plannedTasks, final: pending === 0 });- if (decision !== 'wait')+ if (decision !== 'wait') {+ await rollups?.idle();return { stamped: await act(decision, run, resultCount), lateStamp: null };+ }logger.info({ brandId, runId, resultCount, plannedTasks, pending },'tracking run short of complete — waiting for late results before stamping',);- const lateStamp = awaitLateResults(run, { pollMs, waitMs }).catch((err) => {+ const lateStamp = (async () => {+ await rollups?.flush(resultCount);+ return awaitLateResults(run, { pollMs, waitMs, rollups });+ })().catch((err) => {logger.error({ err, brandId, runId }, 'waiting for late results failed');return false;});diff --git a/server/src/lib/run-settlement.test.js b/server/src/lib/run-settlement.test.jsindex 24e38defc..9ae825d2e 100644--- a/server/src/lib/run-settlement.test.js+++ b/server/src/lib/run-settlement.test.js@@ -139,3 +139,72 @@ describe('settleTrackingRun', () => {await expect(out.lateStamp).resolves.toBe(false);});});++/**+ * A brand's first run hands over its rollup refresher (first-run-rollups.js):+ * the stamp, which refreshes the same days, waits for it, and a run that waits+ * for late answers keeps the rollups moving meanwhile.+ */+describe('settleTrackingRun with a first run refresher', () => {+ /** A refresher whose running refresh ends only when released. */+ function refresher() {+ let release;+ const busy = new Promise((resolve) => {+ release = resolve;+ });+ const calls = [];+ return {+ calls,+ release: () => release(),+ note: vi.fn((landed) => calls.push(['note', landed])),+ idle: vi.fn(async () => {+ calls.push(['idle']);+ await busy;+ }),+ flush: vi.fn(async (landed) => {+ calls.push(['flush', landed]);+ }),+ };+ }+ const tick = () => new Promise((resolve) => setTimeout(resolve, 5));++ it('stamps only once the refresh still running is done', async () => {+ db.results = 98;+ db.pending = 2;+ const rollups = refresher();++ const settling = settleTrackingRun(run, { ...fast, rollups });+ await tick();+ expect(db.stamped).toBeNull();++ rollups.release();+ expect(await settling).toEqual({ stamped: true, lateStamp: null });+ expect(db.stamped.result_count).toBe(98);+ // A run that stamps at once needs no refresh of its own: the stamp's covers it.+ expect(rollups.flush).not.toHaveBeenCalled();+ });++ it('refreshes once when it has to wait, follows the late answers, and stamps once idle', async () => {+ db.results = 54;+ db.pending = 46;+ const rollups = refresher();++ const out = await settleTrackingRun(run, { pollMs: 5, waitMs: 1000, rollups });+ expect(out.stamped).toBe(false);+ await tick();+ db.results = 70;+ await tick();+ db.results = 97;+ db.pending = 3;+ await tick();+ // Complete now, but a refresh is still running.+ expect(db.stamped).toBeNull();++ rollups.release();+ await expect(out.lateStamp).resolves.toBe(true);+ expect(db.stamped.result_count).toBe(97);+ expect(rollups.calls[0]).toEqual(['flush', 54]);+ expect(rollups.calls).toContainEqual(['note', 70]);+ expect(rollups.calls.at(-1)).toEqual(['idle']);+ });+});diff --git a/server/src/routes/internal-trigger-tracking.js b/server/src/routes/internal-trigger-tracking.jsindex 31fe49492..0545a08f8 100644--- a/server/src/routes/internal-trigger-tracking.js+++ b/server/src/routes/internal-trigger-tracking.js@@ -42,7 +42,7 @@ router.post('/', async (req, res) => {try {const { job, created } = await startFirstRun(parsed.data);if (created) {- runTrackingJob(job.id, req.app.get('io'));+ runTrackingJob(job.id, req.app.get('io'), { priority: true });analyzeBrandVolumes(brandId, { onlyMissing: true }).catch((err) => {req.log.error({ err, brandId }, 'first-run volume analysis error');});diff --git a/server/src/routes/internal-trigger-tracking.test.js b/server/src/routes/internal-trigger-tracking.test.jsindex 7178d144a..f45e9a4f9 100644--- a/server/src/routes/internal-trigger-tracking.test.js+++ b/server/src/routes/internal-trigger-tracking.test.js@@ -62,7 +62,8 @@ describe('POST /api/internal/trigger-tracking', () => {expect(res.status).toBe(200);expect(await res.json()).toEqual({ success: true, jobId: 'job1', created: true });expect(startFirstRun).toHaveBeenCalledWith({ brandId: 'b1', since });- expect(runTrackingJob).toHaveBeenCalledWith('job1', io);+ // Ahead of the queue (job-runner.js).+ expect(runTrackingJob).toHaveBeenCalledWith('job1', io, { priority: true });expect(analyzeBrandVolumes).toHaveBeenCalledWith('b1', { onlyMissing: true });});diff --git a/server/src/workers/tracking-worker.first-run.test.js b/server/src/workers/tracking-worker.first-run.test.jsnew file mode 100644index 000000000..0d3fea6ea--- /dev/null+++ b/server/src/workers/tracking-worker.first-run.test.js@@ -0,0 +1,141 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';++/**+ * A brand's first run (no run of it ever stamped), end to end through the+ * worker in webhook mode: its rollups refresh while Cloro's answers land, and+ * the stamp gets its refresher and checks every 30 s. Later runs are as before.+ */++const STARTED = '2026-10-29T10:00:00.000Z';+const db = vi.hoisted(() => ({ stampedRuns: [], pendingPolls: [] }));++vi.mock('../config/supabase.js', () => {+ const answer = (table, ops) => {+ const did = (op) => ops.includes(op);+ switch (table) {+ case 'brands':+ return {+ data: { id: 'b1', name: 'Acme', organization_id: 'o1', shopping_mode_enabled: false },+ };+ case 'brand_domains':+ return { data: [{ domain: 'acme.example' }] };+ case 'prompt_sets':+ return { data: [{ id: 's1' }] };+ case 'prompts':+ return {+ data: [{ id: 'p1', text: 'best x', platforms: ['chatgpt-web', 'copilot-web'] }],+ error: null,+ };+ case 'tracking_runs':+ if (did('insert')) return { data: { id: 'r1', started_at: STARTED }, error: null };+ if (did('not')) return { data: db.stampedRuns, error: null };+ return { error: null };+ case 'cloro_pending_tasks':+ if (did('insert')) return { error: null };+ return { data: db.pendingPolls.shift() ?? [], error: null };+ case 'prompt_results':+ return { data: [{ id: 'result-1' }], error: null };+ default:+ return { data: null, error: null };+ }+ };+ return {+ default: {+ from: (table) => {+ const ops = [];+ const chain = new Proxy(+ {},+ {+ get: (_target, prop) => {+ if (prop === 'then')+ return (resolve, reject) =>+ Promise.resolve(answer(table, ops)).then(resolve, reject);+ if (prop === 'single' || prop === 'maybeSingle')+ return async () => answer(table, ops);+ return () => {+ ops.push(prop);+ return chain;+ };+ },+ },+ );+ return chain;+ },+ },+ };+});++let taskIds = 0;+vi.mock('../lib/cloro-scraper.js', () => ({+ submitScraperTask: vi.fn(async (_text, scraperId) => ({ taskId: `t${++taskIds}`, scraperId })),+ pollScraperResult: vi.fn(),+}));+vi.mock('../lib/cloro-countries.js', () => ({+ loadSupportedCountries: async () => new Map(),+ supportedLocations: (locations) => locations,+}));+const settleTrackingRun = vi.hoisted(() => vi.fn());+vi.mock('../lib/run-settlement.js', () => ({ settleTrackingRun }));+const refreshRunDays = vi.hoisted(() => vi.fn());+vi.mock('../lib/insights-rollups.js', () => ({ refreshRunDays }));+vi.mock('../lib/ai-tracker.js', () => ({ runPrompt: vi.fn(), analyzeSentimentAI: vi.fn() }));+vi.mock('../lib/opportunity-generator.js', () => ({ generateContentOpportunities: vi.fn() }));+vi.mock('../lib/prompt-clusters.js', () => ({ refreshPromptClusters: vi.fn() }));+vi.mock('../lib/site-pages.js', () => ({ refreshSitePages: vi.fn() }));+vi.mock('../lib/target-url-stats.js', () => ({ updateTargetUrlStats: vi.fn() }));+vi.mock('../lib/citation-rows.js', () => ({ persistCitationRows: vi.fn() }));++const { processTrackingJob } = await import('./tracking-worker.js');++const pending = (...ids) => ids.map((id) => ({ task_id: id, submitted_at: STARTED }));++/** A full run, its drain polled every 15 s. */+async function fullRun() {+ const result = processTrackingJob({ brandId: 'b1', source: 'manual', job: { progress() {} } });+ for (let i = 0; i < 10; i++) await vi.advanceTimersByTimeAsync(15_000);+ return result;+}++beforeEach(() => {+ vi.useFakeTimers();+ vi.stubEnv('CLORO_WEBHOOK_URL', 'https://api.example/cloro/callback');+ taskIds = 0;+ settleTrackingRun.mockReset().mockResolvedValue({ stamped: true, lateStamp: null });+ refreshRunDays.mockReset().mockResolvedValue(true);+});++afterEach(() => {+ vi.useRealTimers();+ vi.unstubAllEnvs();+});++describe('processTrackingJob, a brand’s first run', () => {+ it('refreshes the rollups as answers land, and hands the stamp its refresher', async () => {+ db.stampedRuns = [];+ // Two tasks submitted; one answered by the second poll, both by the third.+ db.pendingPolls = [pending('t1', 't2'), pending('t2'), []];++ await fullRun();++ expect(refreshRunDays).toHaveBeenCalledWith('b1', STARTED);+ const [, options] = settleTrackingRun.mock.calls[0];+ expect(options.pollMs).toBe(30_000);+ expect(options.rollups).toEqual(+ expect.objectContaining({+ note: expect.any(Function),+ idle: expect.any(Function),+ flush: expect.any(Function),+ }),+ );+ });++ it('leaves a later run as it was: no refresh mid-run, the usual stamp', async () => {+ db.stampedRuns = [{ id: 'yesterday' }];+ db.pendingPolls = [pending('t1', 't2'), pending('t2'), []];++ await fullRun();++ expect(refreshRunDays).not.toHaveBeenCalled();+ expect(settleTrackingRun.mock.calls[0][1]).toBeUndefined();+ });+});diff --git a/server/src/workers/tracking-worker.js b/server/src/workers/tracking-worker.jsindex ca80ffa20..7233dd69a 100644--- a/server/src/workers/tracking-worker.js+++ b/server/src/workers/tracking-worker.js@@ -23,6 +23,11 @@ import {} from '../lib/locations.js';import { loadSupportedCountries, supportedLocations } from '../lib/cloro-countries.js';import { settleTrackingRun } from '../lib/run-settlement.js';+import {+ FIRST_RUN_LATE_POLL_MS,+ isFirstRun,+ progressiveRollups,+} from '../lib/first-run-rollups.js';import logger from '../lib/logger.js';function resolveModelPlatform(model) {@@ -245,6 +250,12 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,const isFullRun = !promptId && (!promptIds || promptIds.length === 0);let trackingRunId = null;let trackingRunStartedAt = null;+ // A brand's first full run (no run of it ever stamped) fills the rollups as+ // its answers land, and checks for its stamp more often+ // (first-run-rollups.js). Read before this run's own row exists. Not the+ // job's `afterPayment`, which only orders the queue: a new brand's first+ // run need not follow a payment.+ const firstRun = isFullRun && (await isFirstRun(brandId));if (isFullRun) {// A run that died mid-flight (crash, abort) leaves an uncompleted row;// clear those so at most one in-progress row exists per brand.@@ -360,6 +371,16 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,let insertedCount = 0;let completedTasks = 0;+ const firstRunRollups = progressiveRollups({+ brandId,+ startedAt: trackingRunStartedAt,+ enabled: firstRun,+ });+ // Answers landed so far, for the refresher: those this worker stored, and+ // in webhook mode the Cloro tasks answered (the callback stores those).+ let scraperProcessed = 0;+ const landedSoFar = () => insertedCount + scraperProcessed;+async function insertResult(row) {// `created_at` comes back from the insert rather than being stamped here:// the citation rows copy it, and a value computed in app code would drift@@ -374,6 +395,7 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,throw error;}insertedCount++;+ firstRunRollups.note(landedSoFar());// Best-effort: mark target URLs cited by this answer (00032).await updateTargetUrlStats(row.prompt_id, row.citations, new Date().toISOString());// Best-effort: expand the citation array into rows (#732). The jsonb@@ -610,6 +632,8 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,const pending = ourRows.length;pendingAtExit = pending;const processed = expectedSubmitted - pending;+ scraperProcessed = processed;+ firstRunRollups.note(landedSoFar());const allPendingAreOld = allTasksAreStale(ourRows, ghostTaskAgeMs);// "Delivery started" has to mean a result actually landed, not just@@ -925,12 +949,17 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,// would recompute the previous run's window and mail numbers the recipient// already has.const settlement = trackingRunId- ? await settleTrackingRun({- brandId,- runId: trackingRunId,- startedAt: trackingRunStartedAt,- plannedTasks: totalTasks,- })+ ? await settleTrackingRun(+ {+ brandId,+ runId: trackingRunId,+ startedAt: trackingRunStartedAt,+ plannedTasks: totalTasks,+ },+ // A first run also hands over its rollup refresher: the stamp waits+ // for it, and late answers keep the rollups moving.+ firstRun ? { pollMs: FIRST_RUN_LATE_POLL_MS, rollups: firstRunRollups } : undefined,+ ): { stamped: false, lateStamp: null };return {