Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(jobs): restart waiting jobs whose run chain was lost, and keep the slot limit (#945)
Gökhan Geyik091b427b9951
diff --git a/server/src/lib/job-manager.js b/server/src/lib/job-manager.jsindex 96abfa0d6..88e921426 100644--- a/server/src/lib/job-manager.js+++ b/server/src/lib/job-manager.js@@ -77,29 +77,59 @@ export async function failJob(jobId, reason) {}/**- * Mark a job as active and increment attempts.+ * Atomically move a job from 'waiting' to 'active' and count the attempt.+ * Returns the job row, or null when it was not waiting — cancelled, already+ * taken by another run chain, or the read failed. The status guard is what+ * makes it safe for the recovery sweep and a job's own retry timer to race+ * for the same job: only one of them gets it.*/-export async function markJobActive(jobId) {+export async function claimWaitingJob(jobId) {const now = new Date().toISOString();-- // Use rpc-free approach: fetch then update- const { data: job } = await supabaseAdmin+ const { data: job, error } = await supabaseAdmin.from('jobs')- .select('attempts')+ .update({ status: 'active', started_at: now, updated_at: now }).eq('id', jobId)- .single();+ .eq('status', 'waiting')+ .select('*')+ .maybeSingle();- const { error } = await supabaseAdmin+ if (error) {+ logger.error({ err: error, jobId }, 'failed to claim job');+ return null;+ }+ if (!job) return null;++ const attempts = (job.attempts || 0) + 1;+ const { error: attemptsError } = await supabaseAdmin.from('jobs')- .update({- status: 'active',- started_at: now,- updated_at: now,- attempts: (job?.attempts || 0) + 1,- })+ .update({ attempts }).eq('id', jobId);+ if (attemptsError) logger.error({ err: attemptsError, jobId }, 'failed to count job attempt');++ return { ...job, attempts };+}++/**+ * Jobs left in 'waiting' — the recovery sweep's input. Only rows untouched+ * for `idleMs`, so a job created a moment ago is left to the call that+ * created it, and created within `maxAgeMs`, because an older nightly job+ * would duplicate the next night's run.+ */+export async function listWaitingJobs({ idleMs, maxAgeMs }) {+ const now = Date.now();+ const { data, error } = await supabaseAdmin+ .from('jobs')+ .select('id, type')+ .eq('status', 'waiting')+ .lt('updated_at', new Date(now - idleMs).toISOString())+ .gt('created_at', new Date(now - maxAgeMs).toISOString())+ .order('created_at', { ascending: true });- if (error) logger.error({ err: error, jobId }, 'failed to mark job active');+ if (error) {+ logger.error({ err: error }, 'failed to list waiting jobs');+ return [];+ }+ return data ?? [];}/**diff --git a/server/src/lib/job-runner.js b/server/src/lib/job-runner.jsindex ec0836159..7d9616d2d 100644--- a/server/src/lib/job-runner.js+++ b/server/src/lib/job-runner.js@@ -6,7 +6,8 @@import {getJob,- markJobActive,+ claimWaitingJob,+ listWaitingJobs,completeJob,failJob,updateJobProgress,@@ -29,6 +30,19 @@ const MAX_CONCURRENT_TRACKING = Number(process.env.TRACKING_CONCURRENCY) || 2;let activeContentCount = 0;const MAX_CONCURRENT_CONTENT = 2;+// Jobs with a live run chain in this process: waiting for a slot, waiting to+// retry, or running. The queue lives only in memory, so a job whose chain is+// lost — a restart, an error before it got a slot — would sit in 'waiting'+// forever; the recovery sweep restarts those, and this set keeps it from+// starting a second chain for a job this process is still handling.+const queuedJobs = new Set();++// The sweep leaves a job alone for its first minutes (the call that created+// it starts it) and gives up on it after 20h, when the next night's run has+// either started or is about to.+const RECOVERY_IDLE_MS = 2 * 60_000;+const RECOVERY_MAX_AGE_MS = 20 * 60 * 60_000;+/*** Create a mock job object that mirrors the Bull job interface* used by processTrackingJob / processContentJob.@@ -44,22 +58,34 @@ function createJobProxy(jobId, signal) {* Run a tracking job. Call without await (fire-and-forget).* Handles concurrency gating, retry, and Socket.IO emit.*/-export async function runTrackingJob(jobId, io) {+export function runTrackingJob(jobId, io) {+ if (queuedJobs.has(jobId)) return;+ queuedJobs.add(jobId);+ void attemptTrackingJob(jobId, io);+}++async function attemptTrackingJob(jobId, io) {if (activeTrackingCount >= MAX_CONCURRENT_TRACKING) {- setTimeout(() => runTrackingJob(jobId, io), 5000);+ setTimeout(() => attemptTrackingJob(jobId, io), 5000);return;}- const jobRow = await getJob(jobId);- if (!jobRow || jobRow.status === 'cancelled') return;-+ // Take the slot before the first await. Otherwise every timer that fires+ // while the claim is in flight sees the same free slot, and a night once+ // ran nine jobs against a limit of three.activeTrackingCount++;+ const jobRow = await claimWaitingJob(jobId);+ if (!jobRow) {+ activeTrackingCount--;+ queuedJobs.delete(jobId);+ return;+ }+const abortController = new AbortController();registerActiveJob(jobId, abortController);+ let retrying = false;try {- await markJobActive(jobId);-const { brandId, promptId, promptIds, immediate } = jobRow.data;const proxy = createJobProxy(jobId, abortController.signal);@@ -166,35 +192,47 @@ export async function runTrackingJob(jobId, io) {.update({ status: 'waiting', updated_at: new Date().toISOString() }).eq('id', jobId);- setTimeout(() => runTrackingJob(jobId, io), delay);+ retrying = true;+ setTimeout(() => attemptTrackingJob(jobId, io), delay);} else {await failJob(jobId, err.message);}} finally {unregisterActiveJob(jobId);activeTrackingCount--;+ if (!retrying) queuedJobs.delete(jobId);}}/*** Run a content generation job. Call without await (fire-and-forget).*/-export async function runContentJob(jobId, io) {+export function runContentJob(jobId, io) {+ if (queuedJobs.has(jobId)) return;+ queuedJobs.add(jobId);+ void attemptContentJob(jobId, io);+}++async function attemptContentJob(jobId, io) {if (activeContentCount >= MAX_CONCURRENT_CONTENT) {- setTimeout(() => runContentJob(jobId, io), 5000);+ setTimeout(() => attemptContentJob(jobId, io), 5000);return;}- const jobRow = await getJob(jobId);- if (!jobRow || jobRow.status === 'cancelled') return;-+ // Slot before the first await — see attemptTrackingJob.activeContentCount++;+ const jobRow = await claimWaitingJob(jobId);+ if (!jobRow) {+ activeContentCount--;+ queuedJobs.delete(jobId);+ return;+ }+const abortController = new AbortController();registerActiveJob(jobId, abortController);+ let retrying = false;try {- await markJobActive(jobId);-const { brandId, model } = jobRow.data;const proxy = createJobProxy(jobId, abortController.signal);@@ -231,12 +269,37 @@ export async function runContentJob(jobId, io) {.update({ status: 'waiting', updated_at: new Date().toISOString() }).eq('id', jobId);- setTimeout(() => runContentJob(jobId, io), delay);+ retrying = true;+ setTimeout(() => attemptContentJob(jobId, io), delay);} else {await failJob(jobId, err.message);}} finally {unregisterActiveJob(jobId);activeContentCount--;+ if (!retrying) queuedJobs.delete(jobId);+ }+}++/**+ * Restart waiting jobs that have no run chain in this process. Runs at+ * startup and every minute (server.js). Returns how many it restarted.+ */+export async function recoverWaitingJobs(io) {+ const jobs = await listWaitingJobs({+ idleMs: RECOVERY_IDLE_MS,+ maxAgeMs: RECOVERY_MAX_AGE_MS,+ });++ let recovered = 0;+ for (const job of jobs) {+ if (queuedJobs.has(job.id)) continue;+ if (job.type === 'tracking') runTrackingJob(job.id, io);+ else if (job.type === 'content') runContentJob(job.id, io);+ else continue;+ recovered++;}++ if (recovered > 0) logger.warn({ recovered }, 'restarted waiting jobs that had no run chain');+ return recovered;}diff --git a/server/src/lib/job-runner.test.js b/server/src/lib/job-runner.test.jsnew file mode 100644index 000000000..4037677f6--- /dev/null+++ b/server/src/lib/job-runner.test.js@@ -0,0 +1,143 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';++const manager = vi.hoisted(() => ({+ claimWaitingJob: vi.fn(),+ listWaitingJobs: vi.fn(),+ getJob: vi.fn(),+ completeJob: vi.fn(),+ failJob: vi.fn(),+ updateJobProgress: vi.fn(),+ registerActiveJob: vi.fn(),+ unregisterActiveJob: vi.fn(),+}));+const processTrackingJob = vi.hoisted(() => vi.fn());+const processContentJob = vi.hoisted(() => vi.fn());++vi.mock('./job-manager.js', () => manager);+vi.mock('../workers/tracking-worker.js', () => ({ processTrackingJob }));+vi.mock('../workers/content-worker.js', () => ({ processContentJob }));+vi.mock('./pulse/engine.js', () => ({ generatePulseForBrand: vi.fn() }));+vi.mock('./signals/pass.js', () => ({ runFirstSignalPass: vi.fn(), runSignalPass: vi.fn() }));+vi.mock('./logger.js', () => ({+ default: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++const { runTrackingJob, recoverWaitingJobs } = await import('./job-runner.js');++// Default TRACKING_CONCURRENCY.+const MAX = 2;++/** A tracking run that stays in flight until released. */+function heldRuns() {+ const releases = [];+ processTrackingJob.mockImplementation(+ () =>+ new Promise((resolve) => {+ releases.push(() => resolve({ resultCount: 0, stamped: false }));+ }),+ );+ return releases;+}++const flush = () => vi.advanceTimersByTimeAsync(0);++/** Finish every run, including the ones still waiting for a slot. */+async function drain(releases) {+ for (let i = 0; i < 10; i++) {+ releases.forEach((release) => release());+ await vi.advanceTimersByTimeAsync(5000);+ }+}++beforeEach(() => {+ vi.useFakeTimers();+ for (const fn of Object.values(manager)) fn.mockReset();+ processTrackingJob.mockReset();+ processContentJob.mockReset();+ // The claim resolves a tick later, as a network call would.+ manager.claimWaitingJob.mockImplementation(async (id) => {+ await Promise.resolve();+ return { id, attempts: 1, max_attempts: 3, data: { brandId: `brand-${id}` } };+ });+});++// Each test drains its runs, so the module's slot counters start at zero.+afterEach(() => {+ vi.useRealTimers();+});++describe('runTrackingJob', () => {+ it('never runs more than the concurrency limit, even when slots are claimed at once', async () => {+ const releases = heldRuns();+ for (const id of ['a', 'b', 'c', 'd', 'e']) runTrackingJob(id);+ await flush();++ expect(processTrackingJob).toHaveBeenCalledTimes(MAX);++ // A finished run frees one slot, and only one waiting job takes it.+ releases[0]();+ await vi.advanceTimersByTimeAsync(5000);+ expect(processTrackingJob).toHaveBeenCalledTimes(MAX + 1);++ await drain(releases);+ expect(processTrackingJob).toHaveBeenCalledTimes(5);+ });++ it('gives the slot back when the job cannot be claimed', async () => {+ const releases = heldRuns();+ manager.claimWaitingJob.mockResolvedValueOnce(null).mockResolvedValueOnce(null);++ for (const id of ['gone-1', 'gone-2', 'f', 'g']) runTrackingJob(id);+ await vi.advanceTimersByTimeAsync(5000);++ expect(processTrackingJob).toHaveBeenCalledTimes(2);+ await drain(releases);+ });++ it('starts one run chain per job, however often it is asked to', async () => {+ const releases = heldRuns();+ runTrackingJob('dup');+ runTrackingJob('dup');+ await flush();++ expect(manager.claimWaitingJob).toHaveBeenCalledTimes(1);+ await drain(releases);+ });+});++describe('recoverWaitingJobs', () => {+ it('restarts waiting jobs with no run chain here, routed by type', async () => {+ const releases = heldRuns();+ processContentJob.mockResolvedValue({ generated: 0 });+ runTrackingJob('live');+ await flush();++ manager.listWaitingJobs.mockResolvedValue([+ { id: 'live', type: 'tracking' },+ { id: 'lost-tracking', type: 'tracking' },+ { id: 'lost-content', type: 'content' },+ { id: 'other', type: 'export' },+ ]);+ const recovered = await recoverWaitingJobs();+ await flush();++ expect(recovered).toBe(2);+ expect(manager.claimWaitingJob.mock.calls.map(([id]) => id)).toEqual([+ 'live',+ 'lost-tracking',+ 'lost-content',+ ]);+ expect(processContentJob).toHaveBeenCalledTimes(1);+ expect(manager.listWaitingJobs).toHaveBeenCalledWith({+ idleMs: 2 * 60_000,+ maxAgeMs: 20 * 60 * 60_000,+ });+ await drain(releases);+ });++ it('does nothing when no job is waiting', async () => {+ manager.listWaitingJobs.mockResolvedValue([]);+ expect(await recoverWaitingJobs()).toBe(0);+ expect(manager.claimWaitingJob).not.toHaveBeenCalled();+ });+});diff --git a/server/src/server.js b/server/src/server.jsindex d28cc676a..e62770e56 100644--- a/server/src/server.js+++ b/server/src/server.js@@ -19,7 +19,7 @@ import {cleanupOldJobs,cleanupStalePendingTasks,} from './lib/job-manager.js';-import { runTrackingJob } from './lib/job-runner.js';+import { runTrackingJob, recoverWaitingJobs } from './lib/job-runner.js';import { sweepInsightsRollups } from './lib/insights-rollups.js';import { sweepActionValidation } from './lib/action-center/validate.js';import { parseScraperResponse } from './lib/cloro-scraper.js';@@ -612,6 +612,13 @@ server.listen(PORT, async () => {await cleanupStaleJobs();await cleanupStalePendingTasks();+ // The job queue lives in memory: restart waiting jobs whose run chain was+ // lost (a restart, an error before they got a slot), now and every minute.+ const recover = () =>+ recoverWaitingJobs(io).catch((err) => logger.error({ err }, 'waiting-job recovery failed'));+ await recover();+ setInterval(recover, 60_000);+if (!isCloud()) {const schedule = process.env.DAILY_CRON_SCHEDULE || '0 6 * * *';cron.schedule(schedule, async () => {