Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

perf(citations): drain the topic-move rebuild queue every 5 minutes (#1006)

Emdadul Haque Apu7c369f1a131a
diff --git a/server/src/lib/citations-rollups.js b/server/src/lib/citations-rollups.js
index 82a3c3a29..0f1e351d8 100644
--- a/server/src/lib/citations-rollups.js
+++ b/server/src/lib/citations-rollups.js
@@ -88,13 +88,33 @@ export async function refreshCitationHistory(brandId, { pauseMs = 0 } = {}) {
return chunks.length;
}
+/** How often the server drains the rebuild queue outside the nightly sweep. */
+export const REBUILD_QUEUE_SCHEDULE = '*/5 * * * *';
+
+/** The drain in progress, if any; a full-history rebuild takes minutes. */
+let rebuildInFlight = null;
+
/**
* Rebuild the brands queued by a topic move (the trigger on prompts in
* 00118). Topic is stored in the rollup rows, so a prompt moved to another
* topic leaves its history under the old one until this runs. A queue entry
* is cleared only if nothing re-queued the brand while it was rebuilt.
+ *
+ * Called on its own schedule and at the end of the nightly sweep. Runs never
+ * overlap: a call made while one is in progress returns at once with
+ * `skipped: true`, and anything it would have picked up is left for the next.
*/
export async function rebuildQueuedCitationRollups() {
+ if (rebuildInFlight) return { rebuilt: 0, failed: 0, skipped: true };
+ rebuildInFlight = drainRebuildQueue();
+ try {
+ return await rebuildInFlight;
+ } finally {
+ rebuildInFlight = null;
+ }
+}
+
+async function drainRebuildQueue() {
const { data: queued, error } = await supabaseAdmin
.from('citation_rollup_rebuilds')
.select('brand_id, requested_at');
diff --git a/server/src/lib/citations-rollups.test.js b/server/src/lib/citations-rollups.test.js
index d9a7634a2..07b394cf9 100644
--- a/server/src/lib/citations-rollups.test.js
+++ b/server/src/lib/citations-rollups.test.js
@@ -167,4 +167,52 @@ describe('rebuildQueuedCitationRollups', () => {
await expect(rebuildQueuedCitationRollups()).resolves.toEqual({ rebuilt: 0, failed: 1 });
expect(deletes).toBe(0);
});
+
+ it('skips a call made while a run is in progress, then runs again once it ends', async () => {
+ let releaseQueue;
+ let reads = 0;
+ from.mockImplementation(() => ({
+ select: () => {
+ reads++;
+ if (reads > 1) return Promise.resolve({ data: [], error: null });
+ return new Promise((resolve) => {
+ releaseQueue = () => resolve({ data: [], error: null });
+ });
+ },
+ }));
+
+ const first = rebuildQueuedCitationRollups();
+ await expect(rebuildQueuedCitationRollups()).resolves.toEqual({
+ rebuilt: 0,
+ failed: 0,
+ skipped: true,
+ });
+ expect(reads).toBe(1);
+
+ releaseQueue();
+ await expect(first).resolves.toEqual({ rebuilt: 0, failed: 0 });
+ await expect(rebuildQueuedCitationRollups()).resolves.toEqual({ rebuilt: 0, failed: 0 });
+ expect(reads).toBe(2);
+ });
+
+ it('runs again after a run that threw', async () => {
+ from.mockImplementationOnce(() => {
+ throw new Error('client down');
+ });
+ await expect(rebuildQueuedCitationRollups()).rejects.toThrow('client down');
+
+ from.mockImplementation(() => ({
+ select: () => Promise.resolve({ data: [], error: null }),
+ }));
+ await expect(rebuildQueuedCitationRollups()).resolves.toEqual({ rebuilt: 0, failed: 0 });
+ });
+
+ it('costs a single query when the queue is empty', async () => {
+ from.mockImplementation(() => ({
+ select: () => Promise.resolve({ data: [], error: null }),
+ }));
+ await rebuildQueuedCitationRollups();
+ expect(from).toHaveBeenCalledTimes(1);
+ expect(rpc).not.toHaveBeenCalled();
+ });
});
diff --git a/server/src/server.js b/server/src/server.js
index ef88339c1..5814083ee 100644
--- a/server/src/server.js
+++ b/server/src/server.js
@@ -21,6 +21,7 @@ import {
} from './lib/job-manager.js';
import { runTrackingJob, recoverWaitingJobs } from './lib/job-runner.js';
import { sweepInsightsRollups } from './lib/insights-rollups.js';
+import { rebuildQueuedCitationRollups, REBUILD_QUEUE_SCHEDULE } from './lib/citations-rollups.js';
import { sweepActionValidation } from './lib/action-center/validate.js';
import { parseScraperResponse } from './lib/cloro-scraper.js';
import { handleScraperResult } from './lib/cloro-result-handler.js';
@@ -664,6 +665,16 @@ server.listen(PORT, async () => {
await recover();
setInterval(recover, 60_000);
+ // Rebuild the citation rollups of brands whose prompts changed topic
+ // (00118) within minutes, not at the nightly sweep, so the Citations topic
+ // filter catches up soon after a move. An empty queue costs one select, and
+ // the function itself refuses to overlap with a run still in progress.
+ cron.schedule(REBUILD_QUEUE_SCHEDULE, () =>
+ rebuildQueuedCitationRollups().catch((err) =>
+ logger.error({ err }, '[citations-rollups] scheduled rebuild crashed'),
+ ),
+ );
+
if (!isCloud()) {
const schedule = process.env.DAILY_CRON_SCHEDULE || '0 6 * * *';
cron.schedule(schedule, async () => {