Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop held autonomous-job dispatch from re-firing every second (#11752)
[._.]/ Adam Eivyfe5b8f4373cd
diff --git a/server/services/agentLifecycle.blockAndBail.test.js b/server/services/agentLifecycle.blockAndBail.test.jsindex 17c5adcc40..03bc40b8a8 100644--- a/server/services/agentLifecycle.blockAndBail.test.js+++ b/server/services/agentLifecycle.blockAndBail.test.js@@ -656,6 +656,6 @@ describe('runAgentSpawn gates — spawn setup failure', () => {await spawnAgentForTask({ id: 'task-job', metadata: { app: APP, jobId: 'job-7' } });- expect(vi.mocked(cosEvents.emit).mock.calls).toContainEqual(['job:spawn-failed', { jobId: 'job-7' }]);+ expect(vi.mocked(cosEvents.emit).mock.calls).toContainEqual(['job:spawn-failed', { jobId: 'job-7', reason: 'failed' }]);});});diff --git a/server/services/agentLifecycle.js b/server/services/agentLifecycle.jsindex 64497cd1ca..73257c1755 100644--- a/server/services/agentLifecycle.js+++ b/server/services/agentLifecycle.js@@ -129,7 +129,7 @@ async function runAdmittedAgentSpawn(task, permit) {persist: privateSecurity,});if (task.metadata?.jobId) {- cosEvents.emit('job:spawn-failed', { jobId: task.metadata.jobId });+ cosEvents.emit('job:spawn-failed', { jobId: task.metadata.jobId, reason: 'failed' });}return null;} finally {diff --git a/server/services/cos.js b/server/services/cos.jsindex 0ed3178bf7..28a24af94a 100644--- a/server/services/cos.js+++ b/server/services/cos.js@@ -156,7 +156,8 @@ import {unregisterJobSchedules,scheduleNextImprovementCheck,registerSingleJobSchedule,- clearSpawningJob+ clearSpawningJob,+ handleJobSpawnFailed} from './cosJobScheduler.js';// Shared capacity and idle-review admission owner. The engines retain their@@ -1544,13 +1545,11 @@ export async function init() {);});- cosEvents.on('job:spawn-failed', async ({ jobId }) => {- emitLog('warn', `Job spawn failed, re-registering schedule: ${jobId}`, { jobId });- clearSpawningJob(jobId);- await registerSingleJobSchedule(jobId).catch(err =>- console.error(`❌ Failed to re-register job schedule after spawn failure for ${jobId}: ${err.message}`)- );- });+ cosEvents.on('job:spawn-failed', ({ jobId, reason }) =>+ handleJobSpawnFailed(jobId, reason).catch(err =>+ console.error(`❌ Failed to re-arm job schedule after spawn failure for ${jobId}: ${err.message}`)+ )+ );// Event-driven triggers: task/file changes → dequeueNextTask.// The task store (cosTaskStore.js) persists tasks and emits this event; thediff --git a/server/services/cosJobScheduler.js b/server/services/cosJobScheduler.jsindex 983ef2e983..52ebae67db 100644--- a/server/services/cosJobScheduler.js+++ b/server/services/cosJobScheduler.js@@ -215,6 +215,35 @@ export function clearSpawningJob(jobId) {}}+/**+ * Re-arm a job whose task:ready dispatch did not produce an agent. lastRun is+ * only advanced by job:spawned, so re-registering here would compute a past-due+ * fire time and re-fire every second.+ * - 'held': a transient dispatch hold (runner down, capacity, maintenance...);+ * keep the occurrence and retry on the 60s defer.+ * - anything else (a real spawn error): record a skip so the job re-arms at its+ * next real period, like a generation failure.+ */+export async function handleJobSpawnFailed(jobId, reason) {+ clearSpawningJob(jobId);+ if (reason === 'held') {+ emitLog('debug', `Job spawn held, retrying in 60s: ${jobId}`, { jobId });+ scheduleEvent({+ id: `job:${jobId}`,+ type: 'once',+ delayMs: 60000,+ handler: () => executeScheduledJob(jobId),+ metadata: { description: 'Autonomous job (dispatch held)', jobId }+ });+ return;+ }+ emitLog('warn', `Job spawn failed, re-arming at next period: ${jobId}`, { jobId });+ await recordJobGateSkip(jobId).catch(err =>+ console.error(`❌ Failed to record failure-skip for ${jobId}: ${err.message}`)+ );+ await registerSingleJobSchedule(jobId);+}+/*** Execute a scheduled autonomous job and re-register its timer.*/diff --git a/server/services/cosJobScheduler.test.js b/server/services/cosJobScheduler.test.jsindex 25e97aa131..803fec852b 100644--- a/server/services/cosJobScheduler.test.js+++ b/server/services/cosJobScheduler.test.js@@ -110,6 +110,7 @@ import {getScheduledActionReservations,clearSpawningJob,registerSingleJobSchedule,+ handleJobSpawnFailed,} from './cosJobScheduler.js';import { schedule as scheduleEvent, cancel as cancelEvent, getEvent as getScheduledEvent } from './eventScheduler.js';vi.mock('./taskSchedule.js', () => ({ getUpcomingTasks: vi.fn() }));@@ -441,3 +442,34 @@ describe('scheduleNextImprovementCheck — getUpcomingTasks input failure (#7527expect(scheduleEvent).toHaveBeenLastCalledWith(expect.objectContaining({ id: 'cos-improvement-check' }));});});++describe('handleJobSpawnFailed — no 1s re-fire loop after a failed dispatch', () => {+ beforeEach(() => {+ scheduleEvent.mockClear();+ recordJobGateSkip.mockClear();+ generateTaskFromJob.mockClear();+ });++ it('a held dispatch retries on the 60s defer and re-runs the job at most once per minute', async () => {+ await handleJobSpawnFailed('job-held', 'held');++ expect(scheduleEvent).toHaveBeenCalledTimes(1);+ const evt = scheduleEvent.mock.calls[0][0];+ expect(evt).toMatchObject({ id: 'job:job-held', type: 'once', delayMs: 60000 });+ expect(recordJobGateSkip).not.toHaveBeenCalled();++ await evt.handler();+ expect(generateTaskFromJob).toHaveBeenCalledTimes(1);+ clearSpawningJob('job-held');+ });++ it('a spawn error records a gate skip and re-arms via the normal schedule, not a 1s retry', async () => {+ getJob.mockResolvedValueOnce({ id: 'job-err', name: 'job-err', enabled: true, intervalMs: 86_400_000, lastRun: new Date().toISOString() });++ await handleJobSpawnFailed('job-err', 'failed');++ expect(recordJobGateSkip).toHaveBeenCalledWith('job-err');+ expect(scheduleEvent).toHaveBeenCalledTimes(1);+ expect(scheduleEvent.mock.calls[0][0].delayMs).toBeGreaterThan(60000);+ });+});diff --git a/server/services/subAgentSpawner.dispatchHolds.test.js b/server/services/subAgentSpawner.dispatchHolds.test.jsindex d45153cd3c..81ae6aa6de 100644--- a/server/services/subAgentSpawner.dispatchHolds.test.js+++ b/server/services/subAgentSpawner.dispatchHolds.test.js@@ -110,7 +110,7 @@ describe('subAgentSpawner — runner-down hold', () => {spawnAgentForTask.mockRejectedValueOnce(Object.assign(new Error('Held'), { code: 'MAINTENANCE_HELD' }));await dispatch({ id: 'held-task', metadata: { app: 'example-app', jobId: 'example-job' } });expect(releaseAppReviewMarker).toHaveBeenCalledWith('example-app');- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'example-job' });+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'example-job', reason: 'held' });});it('holds a manual/automatic task-ready event before runner preparation', async () => {@@ -157,7 +157,7 @@ describe('subAgentSpawner — runner-down hold', () => {await dispatch({ id: 'cos-4', metadata: { jobId: 'job-7' } });- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-7' });+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-7', reason: 'held' });});it('does not consult the runner in direct mode', async () => {@@ -223,7 +223,7 @@ describe('subAgentSpawner — self-update hold (#4124)', () => {await dispatch({ id: 'cos-u3', metadata: { app: 'some-app', jobId: 'job-9' } });expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-9' });+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-9', reason: 'held' });});it('resumes spawning once the update settles — the hold is not sticky', async () => {@@ -289,7 +289,7 @@ describe('subAgentSpawner — local-endpoint capacity hold (#4834)', () => {await dispatch({ id: 'cos-l3', metadata: { app: 'some-app', jobId: 'job-11' } });expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-11' });+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-11', reason: 'held' });});it('releases the slot even when the spawn throws', async () => {@@ -360,7 +360,7 @@ describe('subAgentSpawner — forge-unreachable hold (#5110)', () => {await dispatch({ id: 'cos-f3', metadata: { openPR: true, app: 'some-app', jobId: 'job-13' } });expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');- expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-13' });+ expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-13', reason: 'held' });});it('holds BEFORE reserving a capacity slot, so a held task never books one', async () => {diff --git a/server/services/subAgentSpawner.js b/server/services/subAgentSpawner.jsindex 9cd9d9ab6a..27ba74ac17 100644--- a/server/services/subAgentSpawner.js+++ b/server/services/subAgentSpawner.js@@ -93,7 +93,7 @@ async function holdTask(task, reason) {emitLog('warn', `Failed to release app review marker for ${task.metadata?.app}: ${err.message}`, { taskId: task.id }));if (task.metadata?.jobId) {- cosEvents.emit('job:spawn-failed', { jobId: task.metadata.jobId });+ cosEvents.emit('job:spawn-failed', { jobId: task.metadata.jobId, reason: 'held' });}}@@ -222,7 +222,7 @@ async function handleTaskReady(task) {emitLog('error', `Failed to spawn agent for task ${task.id}: ${err?.message || err}`, { taskId: task.id });const jobId = task.metadata?.jobId;if (jobId) {- cosEvents.emit('job:spawn-failed', { jobId });+ cosEvents.emit('job:spawn-failed', { jobId, reason: 'failed' });}} finally {localSlot.release();