Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(prompts): report volume progress from the run, and pick it up again on reload (#751)
Gökhan Geyik0be228fcd5f6
diff --git a/server/src/lib/volume-analysis.js b/server/src/lib/volume-analysis.jsindex 1a38a9003..447003a40 100644--- a/server/src/lib/volume-analysis.js+++ b/server/src/lib/volume-analysis.js@@ -33,6 +33,16 @@ export function mapVolumeRow(saved) {*/const MAX_KEYWORDS_PER_REQUEST = 1000;+/** A progress listener is a courtesy to the UI; it never fails the run. */+function report(onProgress, progress) {+ if (!onProgress) return;+ try {+ onProgress(progress);+ } catch (err) {+ logger.warn({ err }, 'volume progress listener threw');+ }+}+/*** Look up every distinct keyword in as few requests as possible.*@@ -304,7 +314,7 @@ export async function getVolumeProgress(brandId) {export async function analyzeBrandVolumes(brandId,- { locationCode, languageCode, force = false, onlyMissing = false } = {},+ { locationCode, languageCode, force = false, onlyMissing = false, onProgress } = {},) {const { data: brand, error: brandError } = await supabaseAdmin.from('brands')@@ -363,6 +373,11 @@ export async function analyzeBrandVolumes(const prepared = [];const keywordStart = Date.now();+ // Nothing reaches prompt_volumes until the lookup at the end, so counting+ // rows would read 0 for the whole of this phase and then jump. Progress is+ // reported from here instead, where the work actually happens.+ report(onProgress, { done: 0, total: promptsToAnalyze.length });+// Keywords first, one prompt at a time — a prompt with saved keywords costs// nothing here, the rest cost one LLM call each. This is now the only// per-prompt work in the run.@@ -379,6 +394,11 @@ export async function analyzeBrandVolumes(logger.error({ err, promptId }, 'keyword extraction failed for prompt');results.push({ promptId, error: err.message });}++ report(onProgress, {+ done: prepared.length + results.length,+ total: promptsToAnalyze.length,+ });}logger.info(diff --git a/server/src/lib/volume-analysis.test.js b/server/src/lib/volume-analysis.test.jsindex eb164951d..0f2093a90 100644--- a/server/src/lib/volume-analysis.test.js+++ b/server/src/lib/volume-analysis.test.js@@ -348,6 +348,68 @@ describe('analyzeBrandVolumes', () => {});});+/**+ * Progress reporting.+ *+ * Batching moved every write to the end of the run, so a progress figure taken+ * from the table sat at zero for the whole wait and then jumped. These pin the+ * report to the work itself, one prompt at a time.+ */+describe('analyzeBrandVolumes progress', () => {+ it('reports the total before any prompt has been worked on', async () => {+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 12 }));+ const seen = [];++ await analyzeBrandVolumes('b1', { onlyMissing: true, onProgress: (p) => seen.push(p) });++ expect(seen[0]).toEqual({ done: 0, total: 12 });+ });++ it('advances once per prompt rather than all at once at the end', async () => {+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 5 }));+ const seen = [];++ await analyzeBrandVolumes('b1', { onlyMissing: true, onProgress: (p) => seen.push(p) });++ expect(seen.map((p) => p.done)).toEqual([0, 1, 2, 3, 4, 5]);+ expect(seen.every((p) => p.total === 5)).toBe(true);+ });++ // A prompt whose keyword extraction failed is still a prompt the run has+ // finished with; leaving it out would stall the count short of the total.+ it('counts a failed prompt as done', async () => {+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 3 }));+ extractIntentKeywords.mockRejectedValueOnce(new Error('model overloaded'));+ const seen = [];++ await analyzeBrandVolumes('b1', { onlyMissing: true, onProgress: (p) => seen.push(p) });++ expect(seen[seen.length - 1]).toEqual({ done: 3, total: 3 });+ });++ it('runs without a listener at all', async () => {+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 2 }));++ expect(await analyzeBrandVolumes('b1', { onlyMissing: true })).toHaveLength(2);+ });++ // The listener belongs to the UI. A broken one must not cost the run, which+ // by then may be most of a quarter-hour of work.+ it('survives a listener that throws', async () => {+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 3 }));++ const results = await analyzeBrandVolumes('b1', {+ onlyMissing: true,+ onProgress: () => {+ throw new Error('listener blew up');+ },+ });++ expect(results).toHaveLength(3);+ expect(results.every((r) => !r.error)).toBe(true);+ });+});+/*** Refreshing re-reads Google volumes for prompts that already have keywords.* It never calls the LLM, so after batching the whole refresh is one requestdiff --git a/server/src/routes/volumes.js b/server/src/routes/volumes.jsindex 03491b8d3..2665eecbe 100644--- a/server/src/routes/volumes.js+++ b/server/src/routes/volumes.js@@ -26,7 +26,7 @@ const router = Router();* the request that starts it, so nothing else would stop a double-click from* starting a second pass over the same prompts.*/-const runningBrands = new Set();+const runningBrands = new Map();/*** Claim the brand, or report that someone already holds it. Claiming and@@ -35,10 +35,14 @@ const runningBrands = new Set();*/function claimVolumeAnalysis(brandId) {if (runningBrands.has(brandId)) return false;- runningBrands.add(brandId);+ runningBrands.set(brandId, { done: 0, total: 0 });return true;}+function trackVolumeProgress(brandId, progress) {+ if (runningBrands.has(brandId)) runningBrands.set(brandId, progress);+}+function releaseVolumeAnalysis(brandId) {runningBrands.delete(brandId);}@@ -189,6 +193,7 @@ router.post('/analyze-batch', requireFeature('prompt_volumes'), async (req, res)languageCode,force,onlyMissing: !force,+ onProgress: (progress) => trackVolumeProgress(brandId, progress),}).then(async (results) => {// Quota counts runs, not prompts, so this stays one row per click@@ -231,7 +236,11 @@ router.post('/analyze-batch', requireFeature('prompt_volumes'), async (req, res)/*** GET /api/volumes/status/:brandId- * Progress of a running analysis: { running, total, analyzed }.+ * Progress of a running analysis: { running, total, analyzed, done }.+ *+ * `done` is what the page counts against: during a run it is the number of+ * prompts whose keywords have been worked out, which is where the minutes go.+ * `analyzed` stays the number of rows on disk, which only moves at the end.** `running` is per-process, so a server restart mid-run reports false while* rows are still being written. The client treats that as "stopped" and shows@@ -244,8 +253,17 @@ router.get('/status/:brandId', requireFeature('prompt_volumes'), async (req, resawait assertBrandAccess(brandId, req.user.id);const { total, analyzed } = await getVolumeProgress(brandId);+ const live = runningBrands.get(brandId);- return res.json({ running: runningBrands.has(brandId), total, analyzed });+ // While a run is going, `analyzed` counts rows that are all written at the+ // very end — it would read 0 for the whole wait and then jump to the total.+ // The live counter tracks the work itself, so it moves throughout.+ return res.json({+ running: Boolean(live),+ total: live?.total || total,+ analyzed,+ done: live ? live.done : analyzed,+ });} catch (error) {if (error instanceof PlanLimitError) {return res.status(error.statusCode).json({diff --git a/server/src/routes/volumes.test.js b/server/src/routes/volumes.test.jsindex c784f6a9e..2f41d65bf 100644--- a/server/src/routes/volumes.test.js+++ b/server/src/routes/volumes.test.js@@ -279,25 +279,67 @@ describe('POST /analyze-batch', () => {});describe('GET /status/:brandId', () => {- it('reports progress and whether a run is in flight', async () => {+ /**+ * Nothing reaches prompt_volumes until the lookup at the very end, so a+ * status built on row counts read 0/100 for the whole ten-minute wait and+ * then jumped straight to 100. The run reports its own progress instead.+ */+ it('counts the work in flight, not the rows on disk', async () => {const run = deferred();analyzeBrandVolumes.mockReturnValue(run.promise);await post({ brandId: 'b1' });- getVolumeProgress.mockResolvedValue({ total: 100, analyzed: 32 });+ const { onProgress } = analyzeBrandVolumes.mock.calls[0][1];+ onProgress({ done: 32, total: 100 });+ getVolumeProgress.mockResolvedValue({ total: 100, analyzed: 0 });+const res = await fetch(`${baseUrl}/status/b1`);- expect(await res.json()).toEqual({ running: true, total: 100, analyzed: 32 });+ expect(await res.json()).toEqual({ running: true, total: 100, analyzed: 0, done: 32 });+ run.resolve([]);+ });++ it('moves as the run reports further progress', async () => {+ const run = deferred();+ analyzeBrandVolumes.mockReturnValue(run.promise);+ await post({ brandId: 'b1' });+ const { onProgress } = analyzeBrandVolumes.mock.calls[0][1];++ const seen = [];+ for (const done of [0, 40, 90]) {+ onProgress({ done, total: 100 });+ seen.push((await (await fetch(`${baseUrl}/status/b1`)).json()).done);+ }++ expect(seen).toEqual([0, 40, 90]);run.resolve([]);});- it('reports a brand with no run as stopped', async () => {+ // Once the run is over the rows are the truth, and they are what a page+ // arriving later has to go on.+ it('falls back to the rows on disk when nothing is running', async () => {getVolumeProgress.mockResolvedValue({ total: 100, analyzed: 100 });expect(await (await fetch(`${baseUrl}/status/b1`)).json()).toEqual({running: false,total: 100,analyzed: 100,+ done: 100,});});++ // The page reads this on load to decide whether to show its button or its+ // spinner; a refresh mid-run must not present the button again.+ it('still reports a run that was started before this request', async () => {+ const run = deferred();+ analyzeBrandVolumes.mockReturnValue(run.promise);+ await post({ brandId: 'b1' });++ expect((await (await fetch(`${baseUrl}/status/b1`)).json()).running).toBe(true);++ run.resolve([]);+ await new Promise((resolve) => setTimeout(resolve, 10));++ expect((await (await fetch(`${baseUrl}/status/b1`)).json()).running).toBe(false);+ });});diff --git a/web/src/app/[locale]/(dashboard)/dashboard/prompts/page.tsx b/web/src/app/[locale]/(dashboard)/dashboard/prompts/page.tsxindex 053c2997f..e4ad29f42 100644--- a/web/src/app/[locale]/(dashboard)/dashboard/prompts/page.tsx+++ b/web/src/app/[locale]/(dashboard)/dashboard/prompts/page.tsx@@ -949,7 +949,7 @@ export default function PromptsPage() {const [loading, setLoading] = useState(true);const [analyzing, setAnalyzing] = useState(false);const [analysisProgress, setAnalysisProgress] = useState<{- analyzed: number;+ done: number;total: number;} | null>(null);// Bumped whenever a run is superseded or the brand changes, so a poll that is@@ -1111,15 +1111,87 @@ export default function PromptsPage() {};}, [loadData]);- // Switching brands (or leaving) ends the watch. The run itself is on the- // server and carries on; only this page's narration of it stops.+ // loadData changes identity whenever the date range does. The watch below+ // must not restart for that, so it reaches the current one through a ref.+ const loadDataRef = useRef(loadData);useEffect(() => {+ loadDataRef.current = loadData;+ }, [loadData]);++ /**+ * Follow a run that is already going.+ *+ * Progress is the server's own count of prompts it has finished with, not+ * rows on disk: batching writes every row in one go at the end, so a row+ * count reads 0 for the whole wait and then jumps straight to the total.+ *+ * Only a status that says the run has stopped counts as finished. Giving up+ * on the watch — deadline reached, status unreadable — says nothing about the+ * run, and claiming otherwise would send someone off with a half-filled table+ * believing it was complete.+ */+ const watchVolumeAnalysis = useCallback(async (brandId: string, gen: number) => {+ let finished = false;+ const deadline = Date.now() + VOLUME_WATCH_MS;++ for (;;) {+ await new Promise((resolve) => setTimeout(resolve, VOLUME_POLL_MS));+ if (analysisGenRef.current !== gen) return { finished: false, superseded: true };++ const status = await getVolumeAnalysisStatus(brandId);+ if (analysisGenRef.current !== gen) return { finished: false, superseded: true };++ if (!status) break;+ setAnalysisProgress({ done: status.done, total: status.total });+ if (!status.running) {+ finished = true;+ break;+ }+ if (Date.now() > deadline) break;+ }++ return { finished, superseded: false };+ }, []);++ /**+ * A run outlives the page that started it, so on arrival — a reload, a tab+ * reopened, a return to this brand — the page has to ask whether one is+ * already going. Without this it offers the Analyze button again while the+ * work is still in flight, which reads as though nothing were happening.+ *+ * Leaving the brand ends the watch. The run itself carries on; only this+ * page's narration of it stops.+ */+ useEffect(() => {+ if (!activeBrandId) return undefined;++ const brandId = activeBrandId;+ const gen = ++analysisGenRef.current;++ (async () => {+ const status = await getVolumeAnalysisStatus(brandId);+ if (analysisGenRef.current !== gen || !status?.running) return;++ setAnalyzing(true);+ setAnalysisProgress({ done: status.done, total: status.total });++ const { finished, superseded } = await watchVolumeAnalysis(brandId, gen);+ if (superseded || analysisGenRef.current !== gen) return;++ setAnalyzing(false);+ setAnalysisProgress(null);+ if (finished) {+ await loadDataRef.current();+ toast.success('Volume analysis finished.');+ }+ })();+return () => {analysisGenRef.current += 1;setAnalyzing(false);setAnalysisProgress(null);};- }, [activeBrandId]);+ }, [activeBrandId, watchVolumeAnalysis]);/*** Start a run, then follow it to the end.@@ -1177,28 +1249,8 @@ export default function PromptsPage() {: 'Volume analysis is already running for this brand.',);- // Only a status that comes back saying the run has stopped counts as- // finished. Giving up on the watch — deadline reached, status unreadable- // — says nothing about the run, and claiming otherwise would send someone- // off with a half-filled table believing it was complete.- let finished = false;- const deadline = Date.now() + VOLUME_WATCH_MS;-- for (;;) {- await new Promise((resolve) => setTimeout(resolve, VOLUME_POLL_MS));- if (analysisGenRef.current !== gen) return;-- const status = await getVolumeAnalysisStatus(activeBrandId);- if (analysisGenRef.current !== gen) return;-- if (!status) break;- setAnalysisProgress({ analyzed: status.analyzed, total: status.total });- if (!status.running) {- finished = true;- break;- }- if (Date.now() > deadline) break;- }+ const { finished, superseded } = await watchVolumeAnalysis(activeBrandId, gen);+ if (superseded) return;await loadData();if (analysisGenRef.current === gen) {@@ -1267,7 +1319,7 @@ export default function PromptsPage() {// Reads "Analyzing 32/100..." once the first poll lands, so a run measured in// minutes shows movement instead of an indefinite spinner.const analyzingLabel = analysisProgress- ? `Analyzing ${analysisProgress.analyzed}/${analysisProgress.total}...`+ ? `Analyzing ${analysisProgress.done}/${analysisProgress.total}...`: 'Analyzing...';// Join prompts with their volume + visibility summaries once, reused by thediff --git a/web/src/lib/actions/volumes.ts b/web/src/lib/actions/volumes.tsindex 40412c287..43e199ea0 100644--- a/web/src/lib/actions/volumes.ts+++ b/web/src/lib/actions/volumes.ts@@ -67,7 +67,10 @@ export type StartVolumeAnalysisResult =export interface VolumeAnalysisStatus {running: boolean;total: number;+ /** Rows on disk. During a run these are all written at the very end. */analyzed: number;+ /** Prompts the run has finished with — what a progress readout counts. */+ done: number;}async function toErrorCode(res: Response): Promise<VolumeErrorCode> {@@ -146,6 +149,7 @@ export async function getVolumeAnalysisStatus(running: Boolean(body.running),total: body.total ?? 0,analyzed: body.analyzed ?? 0,+ done: body.done ?? body.analyzed ?? 0,};}