Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop held autonomous-job dispatch from re-firing every second (#11752)

[._.]/ Adam Eivyfe5b8f4373cd
diff --git a/server/services/agentLifecycle.blockAndBail.test.js b/server/services/agentLifecycle.blockAndBail.test.js
index 17c5adcc40..03bc40b8a8 100644
--- a/server/services/agentLifecycle.blockAndBail.test.js
+++ b/server/services/agentLifecycle.blockAndBail.test.js
@@ -656,6 +656,6 @@ describe('runAgentSpawn gates — spawn setup failure', () => {
await spawnAgentForTask({ id: 'task-job', metadata: { app: APP, jobId: 'job-7' } });
- expect(vi.mocked(cosEvents.emit).mock.calls).toContainEqual(['job:spawn-failed', { jobId: 'job-7' }]);
+ expect(vi.mocked(cosEvents.emit).mock.calls).toContainEqual(['job:spawn-failed', { jobId: 'job-7', reason: 'failed' }]);
});
});
diff --git a/server/services/agentLifecycle.js b/server/services/agentLifecycle.js
index 64497cd1ca..73257c1755 100644
--- a/server/services/agentLifecycle.js
+++ b/server/services/agentLifecycle.js
@@ -129,7 +129,7 @@ async function runAdmittedAgentSpawn(task, permit) {
persist: privateSecurity,
});
if (task.metadata?.jobId) {
- cosEvents.emit('job:spawn-failed', { jobId: task.metadata.jobId });
+ cosEvents.emit('job:spawn-failed', { jobId: task.metadata.jobId, reason: 'failed' });
}
return null;
} finally {
diff --git a/server/services/cos.js b/server/services/cos.js
index 0ed3178bf7..28a24af94a 100644
--- a/server/services/cos.js
+++ b/server/services/cos.js
@@ -156,7 +156,8 @@ import {
unregisterJobSchedules,
scheduleNextImprovementCheck,
registerSingleJobSchedule,
- clearSpawningJob
+ clearSpawningJob,
+ handleJobSpawnFailed
} from './cosJobScheduler.js';
// Shared capacity and idle-review admission owner. The engines retain their
@@ -1544,13 +1545,11 @@ export async function init() {
);
});
- cosEvents.on('job:spawn-failed', async ({ jobId }) => {
- emitLog('warn', `Job spawn failed, re-registering schedule: ${jobId}`, { jobId });
- clearSpawningJob(jobId);
- await registerSingleJobSchedule(jobId).catch(err =>
- console.error(`❌ Failed to re-register job schedule after spawn failure for ${jobId}: ${err.message}`)
- );
- });
+ cosEvents.on('job:spawn-failed', ({ jobId, reason }) =>
+ handleJobSpawnFailed(jobId, reason).catch(err =>
+ console.error(`❌ Failed to re-arm job schedule after spawn failure for ${jobId}: ${err.message}`)
+ )
+ );
// Event-driven triggers: task/file changes → dequeueNextTask.
// The task store (cosTaskStore.js) persists tasks and emits this event; the
diff --git a/server/services/cosJobScheduler.js b/server/services/cosJobScheduler.js
index 983ef2e983..52ebae67db 100644
--- a/server/services/cosJobScheduler.js
+++ b/server/services/cosJobScheduler.js
@@ -215,6 +215,35 @@ export function clearSpawningJob(jobId) {
}
}
+/**
+ * Re-arm a job whose task:ready dispatch did not produce an agent. lastRun is
+ * only advanced by job:spawned, so re-registering here would compute a past-due
+ * fire time and re-fire every second.
+ * - 'held': a transient dispatch hold (runner down, capacity, maintenance...);
+ * keep the occurrence and retry on the 60s defer.
+ * - anything else (a real spawn error): record a skip so the job re-arms at its
+ * next real period, like a generation failure.
+ */
+export async function handleJobSpawnFailed(jobId, reason) {
+ clearSpawningJob(jobId);
+ if (reason === 'held') {
+ emitLog('debug', `Job spawn held, retrying in 60s: ${jobId}`, { jobId });
+ scheduleEvent({
+ id: `job:${jobId}`,
+ type: 'once',
+ delayMs: 60000,
+ handler: () => executeScheduledJob(jobId),
+ metadata: { description: 'Autonomous job (dispatch held)', jobId }
+ });
+ return;
+ }
+ emitLog('warn', `Job spawn failed, re-arming at next period: ${jobId}`, { jobId });
+ await recordJobGateSkip(jobId).catch(err =>
+ console.error(`❌ Failed to record failure-skip for ${jobId}: ${err.message}`)
+ );
+ await registerSingleJobSchedule(jobId);
+}
+
/**
* Execute a scheduled autonomous job and re-register its timer.
*/
diff --git a/server/services/cosJobScheduler.test.js b/server/services/cosJobScheduler.test.js
index 25e97aa131..803fec852b 100644
--- a/server/services/cosJobScheduler.test.js
+++ b/server/services/cosJobScheduler.test.js
@@ -110,6 +110,7 @@ import {
getScheduledActionReservations,
clearSpawningJob,
registerSingleJobSchedule,
+ handleJobSpawnFailed,
} from './cosJobScheduler.js';
import { schedule as scheduleEvent, cancel as cancelEvent, getEvent as getScheduledEvent } from './eventScheduler.js';
vi.mock('./taskSchedule.js', () => ({ getUpcomingTasks: vi.fn() }));
@@ -441,3 +442,34 @@ describe('scheduleNextImprovementCheck — getUpcomingTasks input failure (#7527
expect(scheduleEvent).toHaveBeenLastCalledWith(expect.objectContaining({ id: 'cos-improvement-check' }));
});
});
+
+describe('handleJobSpawnFailed — no 1s re-fire loop after a failed dispatch', () => {
+ beforeEach(() => {
+ scheduleEvent.mockClear();
+ recordJobGateSkip.mockClear();
+ generateTaskFromJob.mockClear();
+ });
+
+ it('a held dispatch retries on the 60s defer and re-runs the job at most once per minute', async () => {
+ await handleJobSpawnFailed('job-held', 'held');
+
+ expect(scheduleEvent).toHaveBeenCalledTimes(1);
+ const evt = scheduleEvent.mock.calls[0][0];
+ expect(evt).toMatchObject({ id: 'job:job-held', type: 'once', delayMs: 60000 });
+ expect(recordJobGateSkip).not.toHaveBeenCalled();
+
+ await evt.handler();
+ expect(generateTaskFromJob).toHaveBeenCalledTimes(1);
+ clearSpawningJob('job-held');
+ });
+
+ it('a spawn error records a gate skip and re-arms via the normal schedule, not a 1s retry', async () => {
+ getJob.mockResolvedValueOnce({ id: 'job-err', name: 'job-err', enabled: true, intervalMs: 86_400_000, lastRun: new Date().toISOString() });
+
+ await handleJobSpawnFailed('job-err', 'failed');
+
+ expect(recordJobGateSkip).toHaveBeenCalledWith('job-err');
+ expect(scheduleEvent).toHaveBeenCalledTimes(1);
+ expect(scheduleEvent.mock.calls[0][0].delayMs).toBeGreaterThan(60000);
+ });
+});
diff --git a/server/services/subAgentSpawner.dispatchHolds.test.js b/server/services/subAgentSpawner.dispatchHolds.test.js
index d45153cd3c..81ae6aa6de 100644
--- a/server/services/subAgentSpawner.dispatchHolds.test.js
+++ b/server/services/subAgentSpawner.dispatchHolds.test.js
@@ -110,7 +110,7 @@ describe('subAgentSpawner — runner-down hold', () => {
spawnAgentForTask.mockRejectedValueOnce(Object.assign(new Error('Held'), { code: 'MAINTENANCE_HELD' }));
await dispatch({ id: 'held-task', metadata: { app: 'example-app', jobId: 'example-job' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('example-app');
- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'example-job' });
+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'example-job', reason: 'held' });
});
it('holds a manual/automatic task-ready event before runner preparation', async () => {
@@ -157,7 +157,7 @@ describe('subAgentSpawner — runner-down hold', () => {
await dispatch({ id: 'cos-4', metadata: { jobId: 'job-7' } });
- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-7' });
+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-7', reason: 'held' });
});
it('does not consult the runner in direct mode', async () => {
@@ -223,7 +223,7 @@ describe('subAgentSpawner — self-update hold (#4124)', () => {
await dispatch({ id: 'cos-u3', metadata: { app: 'some-app', jobId: 'job-9' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');
- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-9' });
+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-9', reason: 'held' });
});
it('resumes spawning once the update settles — the hold is not sticky', async () => {
@@ -289,7 +289,7 @@ describe('subAgentSpawner — local-endpoint capacity hold (#4834)', () => {
await dispatch({ id: 'cos-l3', metadata: { app: 'some-app', jobId: 'job-11' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');
- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-11' });
+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-11', reason: 'held' });
});
it('releases the slot even when the spawn throws', async () => {
@@ -360,7 +360,7 @@ describe('subAgentSpawner — forge-unreachable hold (#5110)', () => {
await dispatch({ id: 'cos-f3', metadata: { openPR: true, app: 'some-app', jobId: 'job-13' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');
- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-13' });
+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-13', reason: 'held' });
});
it('holds BEFORE reserving a capacity slot, so a held task never books one', async () => {
diff --git a/server/services/subAgentSpawner.js b/server/services/subAgentSpawner.js
index 9cd9d9ab6a..27ba74ac17 100644
--- a/server/services/subAgentSpawner.js
+++ b/server/services/subAgentSpawner.js
@@ -93,7 +93,7 @@ async function holdTask(task, reason) {
emitLog('warn', `Failed to release app review marker for ${task.metadata?.app}: ${err.message}`, { taskId: task.id })
);
if (task.metadata?.jobId) {
- cosEvents.emit('job:spawn-failed', { jobId: task.metadata.jobId });
+ cosEvents.emit('job:spawn-failed', { jobId: task.metadata.jobId, reason: 'held' });
}
}
@@ -222,7 +222,7 @@ async function handleTaskReady(task) {
emitLog('error', `Failed to spawn agent for task ${task.id}: ${err?.message || err}`, { taskId: task.id });
const jobId = task.metadata?.jobId;
if (jobId) {
- cosEvents.emit('job:spawn-failed', { jobId });
+ cosEvents.emit('job:spawn-failed', { jobId, reason: 'failed' });
}
} finally {
localSlot.release();