Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(tracking): stop progress bar freezing on webhook-mode runs (#279)
Gökhan Geyik0f4cc6014d2a
diff --git a/server/src/lib/job-manager.js b/server/src/lib/job-manager.jsindex 92b5ddf7f..8c51c6f86 100644--- a/server/src/lib/job-manager.js+++ b/server/src/lib/job-manager.js@@ -181,3 +181,24 @@ export async function cleanupOldJobs() {console.error('[job-manager] Failed to cleanup old jobs:', error.message);}}++/**+ * Delete orphaned Cloro pending tasks — rows for which Cloro never delivered a+ * webhook. They sit forever otherwise; left unchecked they accumulate and (with+ * a brand-wide count) used to poison every future run's drain loop. Two hours is+ * well past the worker's 60-minute drain deadline and any realistic Cloro+ * delivery window, so anything older is safe to drop.+ */+export async function cleanupStalePendingTasks() {+ const cutoff = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString();+ const { count, error } = await supabaseAdmin+ .from('cloro_pending_tasks')+ .delete({ count: 'exact' })+ .lt('submitted_at', cutoff);++ if (error) {+ console.error('[job-manager] Failed to cleanup stale pending tasks:', error.message);+ } else if (count && count > 0) {+ console.log(`[job-manager] Cleaned up ${count} orphaned Cloro pending tasks`);+ }+}diff --git a/server/src/server.js b/server/src/server.jsindex f7ad29f0f..9ab0ff76c 100644--- a/server/src/server.js+++ b/server/src/server.js@@ -13,7 +13,12 @@ import { apiLimiter } from './middleware/rate-limiter.js';import requestIdMiddleware from './middleware/request-id.js';import routes from './routes/index.js';import trafficRoutes from './routes/traffic.js';-import { createJob, cleanupStaleJobs, cleanupOldJobs } from './lib/job-manager.js';+import {+ createJob,+ cleanupStaleJobs,+ cleanupOldJobs,+ cleanupStalePendingTasks,+} from './lib/job-manager.js';import { runTrackingJob } from './lib/job-runner.js';import { parseScraperResponse } from './lib/cloro-scraper.js';import { handleScraperResult } from './lib/cloro-result-handler.js';@@ -98,6 +103,10 @@ app.set('io', io);async function runDailyTracking() {const isCloudMode = isCloud();+ // Sweep orphaned Cloro pending tasks daily (runs in both cloud and+ // self-hosted, since both paths funnel through here).+ await cleanupStalePendingTasks();+const { data: brands, error } = await supabaseAdmin.from('brands').select('id, organization_id');if (error || !brands || brands.length === 0) {@@ -387,6 +396,7 @@ server.listen(PORT, async () => {console.log(`AEO Server running on port ${PORT} [${process.env.NODE_ENV}]`);await cleanupStaleJobs();+ await cleanupStalePendingTasks();if (!isCloud()) {const schedule = process.env.DAILY_CRON_SCHEDULE || '0 6 * * *';diff --git a/server/src/workers/tracking-worker.js b/server/src/workers/tracking-worker.jsindex afc6d0ee1..eef65cb6b 100644--- a/server/src/workers/tracking-worker.js+++ b/server/src/workers/tracking-worker.js@@ -190,39 +190,78 @@ export async function processTrackingJob({ brandId, promptId, promptIds, job })}}- // Wait for the webhook handler to drain cloro_pending_tasks for this brand.- // The worker stays alive (cheap DB poll) so the job's `active` status drives- // the UI loading banner until results actually arrive. Hard cap at 60 minutes- // so a stuck Cloro queue doesn't keep workers running indefinitely.- const drainDeadline = Date.now() + 60 * 60 * 1000;- const drainPollMs = 15_000;- const expectedSubmitted = submitted.length;-- while (Date.now() < drainDeadline) {- const { count: remaining } = await supabaseAdmin- .from('cloro_pending_tasks')- .select('*', { count: 'exact', head: true })- .eq('brand_id', brandId);+ // Wait for the webhook handler to drain THIS job's pending tasks. The+ // worker stays alive (cheap DB poll) so the job's `active` status drives+ // the UI loading banner until results actually arrive.+ //+ // We count only the task_ids THIS run submitted — not every pending row+ // for the brand. A brand-wide count is poisoned by orphan rows from tasks+ // Cloro never delivered a webhook for (and by concurrent runs), so it+ // never reaches zero: the drain loop runs to the deadline and the progress+ // bar freezes partway even though results keep landing. Counting our own+ // task_ids lets the loop finish as soon as this run's results are in.+ const submittedTaskIds = new Set(submitted.map((s) => s.taskId));+ const expectedSubmitted = submittedTaskIds.size;++ if (expectedSubmitted > 0) {+ // Hard cap so a stuck Cloro queue can't keep a worker alive forever.+ const drainDeadline = Date.now() + 60 * 60 * 1000;+ const drainPollMs = 15_000;+ // Give up early if delivery stalls — no new result for this many+ // consecutive polls (~10 min) means the rest were almost certainly+ // dropped, so don't hold the bar (and a concurrency slot) for an hour.+ const stallPollLimit = 40;++ let lastPending = expectedSubmitted;+ let stalledPolls = 0;++ while (Date.now() < drainDeadline) {+ // Brand-scoped read (a handful of rows at most), intersected in memory+ // with our own task_ids — avoids a giant `.in(...)` URL.+ const { data: rows, error: drainErr } = await supabaseAdmin+ .from('cloro_pending_tasks')+ .select('task_id')+ .eq('brand_id', brandId);++ // A transient read failure must NOT be read as "0 pending" — that would+ // break the loop early and report the run as finished while tasks are+ // still in flight. Skip this tick and retry on the next poll.+ if (drainErr) {+ console.warn(`[tracking] Pending-task poll failed, retrying: ${drainErr.message}`);+ await new Promise((r) => setTimeout(r, drainPollMs));+ continue;+ }- const pending = remaining ?? 0;- const processed = expectedSubmitted - pending;+ const pending = (rows || []).filter((r) => submittedTaskIds.has(r.task_id)).length;+ const processed = expectedSubmitted - pending;- if (job) {- job.progress({- current: completedTasks + processed,- total: totalTasks,- promptText:- pending > 0- ? `Receiving platform results — ${pending} task(s) still processing...`- : 'All platform results received',- model: null,- platform: 'cloro',- });- }+ if (job) {+ job.progress({+ current: completedTasks + processed,+ total: totalTasks,+ promptText:+ pending > 0+ ? `Receiving platform results — ${pending} task(s) still processing...`+ : 'All platform results received',+ model: null,+ platform: 'cloro',+ });+ }- if (pending === 0) break;+ if (pending === 0) break;- await new Promise((r) => setTimeout(r, drainPollMs));+ if (pending < lastPending) {+ lastPending = pending;+ stalledPolls = 0;+ } else if (++stalledPolls >= stallPollLimit) {+ console.warn(+ `[tracking] Cloro delivery stalled for brand ${brandId} — ${pending} of ${expectedSubmitted} task(s) never returned; continuing.`,+ );+ break;+ }++ await new Promise((r) => setTimeout(r, drainPollMs));+ }}completedTasks += expectedSubmitted;diff --git a/web/src/lib/tracking-job-store.ts b/web/src/lib/tracking-job-store.tsindex d6da58545..1baba307b 100644--- a/web/src/lib/tracking-job-store.ts+++ b/web/src/lib/tracking-job-store.ts@@ -17,7 +17,9 @@ export function loadTrackingJob(): TrackingJob | null {const raw = localStorage.getItem(TRACKING_STORAGE_KEY);if (!raw) return null;const job = JSON.parse(raw) as TrackingJob;- if (Date.now() - job.startedAt > 10 * 60 * 1000) {+ // Large brands legitimately take ~45 min to finish; keep restoring the+ // progress banner across refreshes until the worker's 60-min drain deadline.+ if (Date.now() - job.startedAt > 60 * 60 * 1000) {localStorage.removeItem(TRACKING_STORAGE_KEY);return null;}