Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(cos): advance job schedule on execution failure to prevent 1s re-fire loop

[._.]/ Adam Eivy41aedc43d8d1
diff --git a/server/services/cosJobScheduler.js b/server/services/cosJobScheduler.js
index fc3d53a7c8..983ef2e983 100644
--- a/server/services/cosJobScheduler.js
+++ b/server/services/cosJobScheduler.js
@@ -350,6 +350,11 @@ export async function executeScheduledJob(jobId) {
.catch(err => console.error(`❌ Failed to record CoS budget usage for script job ${job.id}: ${err.message}`));
releaseReservation(); // recorded usage now owns the in-flight count for this action
if (scriptOk) emitLog('info', `Script job executed: ${job.name}`, { jobId: job.id });
+ else {
+ await recordJobGateSkip(jobId).catch(err =>
+ console.error(`❌ Failed to record failure-skip for ${jobId}: ${err.message}`)
+ );
+ }
} else if (isShellJob(job)) {
const startedAt = Date.now();
const shellOk = await executeShellJob(job).then(() => true, err => {
@@ -365,6 +370,11 @@ export async function executeScheduledJob(jobId) {
.catch(err => console.error(`❌ Failed to record CoS budget usage for shell job ${job.id}: ${err.message}`));
releaseReservation(); // recorded usage now owns the in-flight count for this action
if (shellOk) emitLog('info', `Shell job executed: ${job.name}`, { jobId: job.id });
+ else {
+ await recordJobGateSkip(jobId).catch(err =>
+ console.error(`❌ Failed to record failure-skip for ${jobId}: ${err.message}`)
+ );
+ }
} else {
// Check if this job is already being spawned or has a running agent.
// Don't re-register the timer here — the job:spawned handler will do it
@@ -427,6 +437,9 @@ export async function executeScheduledJob(jobId) {
} catch (err) {
clearSpawningJob(jobId);
emitLog('error', `Failed to fire autonomous job: ${job.name} - ${err?.message || err}`, { jobId, category: job.category });
+ await recordJobGateSkip(jobId).catch(skipErr =>
+ console.error(`❌ Failed to record failure-skip for ${jobId}: ${skipErr.message}`)
+ );
}
}
diff --git a/server/services/cosJobScheduler.test.js b/server/services/cosJobScheduler.test.js
index 97961f785a..25e97aa131 100644
--- a/server/services/cosJobScheduler.test.js
+++ b/server/services/cosJobScheduler.test.js
@@ -78,14 +78,16 @@ vi.mock('./cosTaskGenerator.js', () => ({ queueEligibleImprovementTasks: vi.fn()
const generateTaskFromJob = vi.fn(async (job) => ({ id: `task-${job.id}`, description: job.name }));
const executeScriptJob = vi.fn().mockResolvedValue(undefined);
+const executeShellJob = vi.fn().mockResolvedValue(undefined);
+const recordJobGateSkip = vi.fn().mockResolvedValue(undefined);
const checkJobGate = vi.fn().mockResolvedValue({ shouldRun: true, reason: 'ok' });
vi.mock('./autonomousJobs.js', () => ({
generateTaskFromJob: (...a) => generateTaskFromJob(...a),
- recordJobGateSkip: vi.fn().mockResolvedValue(undefined),
+ recordJobGateSkip: (...a) => recordJobGateSkip(...a),
isScriptJob: (job) => job?.type === 'script',
executeScriptJob: (...a) => executeScriptJob(...a),
isShellJob: (job) => job?.type === 'shell',
- executeShellJob: vi.fn().mockResolvedValue(undefined),
+ executeShellJob: (...a) => executeShellJob(...a),
// Dynamically imported inside executeScheduledJob — return a job built from the
// requested id so two distinct ids model two distinct same-tick jobs.
getJob: vi.fn(async (jobId) => ({
@@ -122,6 +124,8 @@ beforeEach(() => {
globalThis.__nextJobType = 'agent';
generateTaskFromJob.mockClear();
executeScriptJob.mockClear();
+ executeShellJob.mockClear();
+ recordJobGateSkip.mockClear();
checkJobGate.mockReset();
checkJobGate.mockResolvedValue({ shouldRun: true, reason: 'ok' });
// maxActionsPerDay:1, nothing recorded yet — the exact same-tick boundary.
@@ -138,8 +142,11 @@ afterEach(() => {
// module-level set and timer don't leak across tests.
clearSpawningJob('job-a');
clearSpawningJob('job-b');
+ clearSpawningJob('job-fail');
clearSpawningJob('script-a');
clearSpawningJob('script-b');
+ clearSpawningJob('script-fail');
+ clearSpawningJob('shell-fail');
delete globalThis.__nextJobType;
});
@@ -229,6 +236,31 @@ describe('executeScheduledJob — same-tick daily-action budget race (#984)', ()
});
});
+describe('executeScheduledJob — execution failure advances schedule to avoid rapid loop', () => {
+ it('calls recordJobGateSkip when generateTaskFromJob throws', async () => {
+ generateTaskFromJob.mockRejectedValueOnce(new Error('Prompt template broken'));
+ await executeScheduledJob('job-fail');
+ expect(recordJobGateSkip).toHaveBeenCalledWith('job-fail');
+ expect(getScheduledActionReservations()).toBe(0);
+ });
+
+ it('calls recordJobGateSkip when executeScriptJob fails', async () => {
+ globalThis.__nextJobType = 'script';
+ executeScriptJob.mockRejectedValueOnce(new Error('Syntax error in job script'));
+ await executeScheduledJob('script-fail');
+ expect(recordJobGateSkip).toHaveBeenCalledWith('script-fail');
+ expect(getScheduledActionReservations()).toBe(0);
+ });
+
+ it('calls recordJobGateSkip when executeShellJob fails', async () => {
+ globalThis.__nextJobType = 'shell';
+ executeShellJob.mockRejectedValueOnce(new Error('Command failed: exit 1'));
+ await executeScheduledJob('shell-fail');
+ expect(recordJobGateSkip).toHaveBeenCalledWith('shell-fail');
+ expect(getScheduledActionReservations()).toBe(0);
+ });
+});
+
describe('registerSingleJobSchedule', () => {
it('cancels a disabled job instead of registering a recurring timer', async () => {
scheduleEvent.mockClear();