Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

18.5 KB
Raw Blame
/**
 * Tests for cosJobScheduler.js — the same-tick daily-action budget race (#984).
 *
 * The per-domain CoS daily action budget is enforced in executeScheduledJob from
 * recorded usage + an in-flight count. Before #984 there was a narrow same-process
 * window: a second scheduled job that read the budget gate while a first job was
 * parked AFTER passing the gate but BEFORE being counted could also pass — agent
 * jobs reserve `spawningJobIds` only after the `checkJobGate` await, and inline
 * script/shell jobs record usage only post-execution. A `maxActionsPerDay: 1` cap
 * could therefore be overshot by one. #984 adds a synchronous reservation counter,
 * bumped the instant a job is admitted to fire (before any further await) and
 * included in the gate's in-flight term, so a second job in that window sees the
 * reservation and is withheld.
 *
 * Unlike the source-level regression guards in cos.test.js, these tests exercise
 * the REAL executeScheduledJob (the race lives in the await interleaving, which a
 * pure inline copy can't reproduce). We deterministically reproduce the window by
 * parking the first job at its first post-reservation await, then running the
 * second job to completion while the first sits in the window. The only un-mocked
 * dependency is the pure `remainingActionBudget` math — everything with I/O is
 * mocked.
 */

import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';

let daemonRunning = true;
const sharedState = {
  paused: false,
  config: { autonomousJobsEnabled: true, maxConcurrentAgents: 5 },
  agents: {},
};

// A manually-resolved deferred — lets a test park one call at a chosen await and
// release it after the other call has run, modelling the exact race interleaving.
function deferred() {
  let resolve;
  const promise = new Promise((r) => { resolve = r; });
  return { promise, resolve };
}

vi.mock('./eventScheduler.js', () => ({
  schedule: vi.fn(),
  cancel: vi.fn(),
  getEvent: vi.fn(() => null),
  parseCronToNextRun: vi.fn(() => new Date(0)),
}));
vi.mock('../lib/timezone.js', () => ({
  getLocalParts: vi.fn(() => ({ dayOfWeek: 1 })),
  nextLocalTime: vi.fn((ms) => ms),
}));
vi.mock('./userTimezone.js', () => ({
  getUserTimezone: vi.fn().mockResolvedValue('UTC'),
}));
vi.mock('../lib/fileUtils.js', async (importOriginal) => ({
  ...(await importOriginal()),
  formatDuration: vi.fn(() => '1h'),
}));
vi.mock('./cosState.js', () => ({
  loadState: vi.fn(async () => sharedState),
  isDaemonRunning: vi.fn(() => daemonRunning),
  canQueueImprovementTasks: vi.fn(() => false),
}));
vi.mock('../lib/domainAutonomy.js', () => ({
  getDomainMode: vi.fn(() => 'execute'),
  DOMAIN_IDS: ['brain', 'memory', 'cos', 'messages'],
}));
// remainingActionBudget is pure — keep it REAL so the gate math is the math.
vi.mock('./domainUsage.js', () => ({
  getDomainBudgetStatus: vi.fn(),
  recordDomainUsage: vi.fn().mockResolvedValue(undefined),
}));
vi.mock('./cosEvents.js', () => ({
  cosEvents: { emit: vi.fn() },
  emitLog: vi.fn(),
}));
vi.mock('./cosTaskStore.js', () => ({ getCosTasks: vi.fn() }));
vi.mock('./cosTaskGenerator.js', () => ({ queueEligibleImprovementTasks: vi.fn() }));

const generateTaskFromJob = vi.fn(async (job) => ({ id: `task-${job.id}`, description: job.name }));
const executeScriptJob = vi.fn().mockResolvedValue(undefined);
const executeShellJob = vi.fn().mockResolvedValue(undefined);
const recordJobGateSkip = vi.fn().mockResolvedValue(undefined);
const checkJobGate = vi.fn().mockResolvedValue({ shouldRun: true, reason: 'ok' });
vi.mock('./autonomousJobs.js', () => ({
  generateTaskFromJob: (...a) => generateTaskFromJob(...a),
  recordJobGateSkip: (...a) => recordJobGateSkip(...a),
  isScriptJob: (job) => job?.type === 'script',
  executeScriptJob: (...a) => executeScriptJob(...a),
  isShellJob: (job) => job?.type === 'shell',
  executeShellJob: (...a) => executeShellJob(...a),
  // Dynamically imported inside executeScheduledJob — return a job built from the
  // requested id so two distinct ids model two distinct same-tick jobs.
  getJob: vi.fn(async (jobId) => ({
    id: jobId,
    name: jobId,
    enabled: true,
    category: 'test',
    intervalMs: 86_400_000,
    type: globalThis.__nextJobType || 'agent',
  })),
}));
vi.mock('./jobGates.js', () => ({
  checkJobGate: (...a) => checkJobGate(...a),
  hasGate: vi.fn(() => false),
}));

import {
  scheduleNextImprovementCheck,
  executeScheduledJob,
  getScheduledActionReservations,
  clearSpawningJob,
  registerSingleJobSchedule,
  handleJobSpawnFailed,
} from './cosJobScheduler.js';
import { schedule as scheduleEvent, cancel as cancelEvent, getEvent as getScheduledEvent } from './eventScheduler.js';
vi.mock('./taskSchedule.js', () => ({ getUpcomingTasks: vi.fn() }));
import { getUpcomingTasks } from './taskSchedule.js';
import { getJob } from './autonomousJobs.js';
import { getDomainBudgetStatus } from './domainUsage.js';
import { canQueueImprovementTasks } from './cosState.js';

beforeEach(() => {
  daemonRunning = true;
  sharedState.agents = {};
  globalThis.__nextJobType = 'agent';
  generateTaskFromJob.mockClear();
  executeScriptJob.mockClear();
  executeShellJob.mockClear();
  recordJobGateSkip.mockClear();
  checkJobGate.mockReset();
  checkJobGate.mockResolvedValue({ shouldRun: true, reason: 'ok' });
  // maxActionsPerDay:1, nothing recorded yet — the exact same-tick boundary.
  getDomainBudgetStatus.mockResolvedValue({
    withinBudget: true,
    exceeded: null,
    budget: { maxActionsPerDay: 1, maxMinutesPerDay: null },
    usage: { actions: 0, ms: 0 },
  });
});

afterEach(() => {
  // Success paths leave the job in spawningJobIds (+ a 5-min timeout); clear so the
  // module-level set and timer don't leak across tests.
  clearSpawningJob('job-a');
  clearSpawningJob('job-b');
  clearSpawningJob('job-fail');
  clearSpawningJob('script-a');
  clearSpawningJob('script-b');
  clearSpawningJob('script-fail');
  clearSpawningJob('shell-fail');
  delete globalThis.__nextJobType;
});

describe('executeScheduledJob — same-tick daily-action budget race (#984)', () => {
  it('a second AGENT job in the post-gate window is withheld (maxActionsPerDay:1)', async () => {
    // Park job-a at checkJobGate — its FIRST await after the budget gate passed
    // and (with the fix) after the synchronous reservation was taken, but BEFORE
    // addSpawningJob counts it. Distinct ids so the `spawningJobIds.has` guard
    // can't be what stops job-b — only the reservation gate should.
    const gateReached = deferred();
    const releaseGate = deferred();
    checkJobGate.mockImplementation(async (jobId) => {
      if (jobId === 'job-a') {
        gateReached.resolve();
        await releaseGate.promise;
      }
      return { shouldRun: true, reason: 'ok' };
    });

    const pA = executeScheduledJob('job-a');
    await gateReached.promise; // job-a now sits in the race window

    // job-b reads the budget gate while job-a is parked. Buggy code: remaining is
    // still 1 (job-a not yet in spawningJobIds), so job-b fires too → two spawns.
    // Fixed code: job-b sees the reservation, is over budget, and is withheld.
    await executeScheduledJob('job-b');

    releaseGate.resolve();
    await pA;

    expect(generateTaskFromJob).toHaveBeenCalledTimes(1);
    expect(generateTaskFromJob).toHaveBeenCalledWith(expect.objectContaining({ id: 'job-a' }));
  });

  it('a second SCRIPT job in the post-record window is withheld (maxActionsPerDay:1)', async () => {
    globalThis.__nextJobType = 'script';
    // Script jobs reach their first post-reservation await at executeScriptJob
    // (recordDomainUsage runs only after it resolves). Park job-a there.
    const execReached = deferred();
    const releaseExec = deferred();
    executeScriptJob.mockImplementation(async (job) => {
      if (job.id === 'script-a') {
        execReached.resolve();
        await releaseExec.promise;
      }
    });

    const pA = executeScheduledJob('script-a');
    await execReached.promise;

    await executeScheduledJob('script-b');

    releaseExec.resolve();
    await pA;

    expect(executeScriptJob).toHaveBeenCalledTimes(1);
    expect(executeScriptJob).toHaveBeenCalledWith(expect.objectContaining({ id: 'script-a' }));
  });

  it('releases the reservation on every exit path (counter settles to zero)', async () => {
    const gateReached = deferred();
    const releaseGate = deferred();
    checkJobGate.mockImplementation(async (jobId) => {
      if (jobId === 'job-a') {
        gateReached.resolve();
        await releaseGate.promise;
      }
      return { shouldRun: true, reason: 'ok' };
    });

    const pA = executeScheduledJob('job-a');
    await gateReached.promise;
    // While job-a is parked the reservation is live (the window the gate read).
    expect(getScheduledActionReservations()).toBe(1);

    await executeScheduledJob('job-b'); // withheld path releases its reservation
    releaseGate.resolve();
    await pA; // handoff to spawningJobIds releases job-a's reservation

    expect(getScheduledActionReservations()).toBe(0);
  });

  it('a single job still fires when the budget has room for exactly one', async () => {
    await executeScheduledJob('job-a');
    expect(generateTaskFromJob).toHaveBeenCalledTimes(1);
    expect(getScheduledActionReservations()).toBe(0);
  });
});

describe('executeScheduledJob — execution failure advances schedule to avoid rapid loop', () => {
  it('calls recordJobGateSkip when generateTaskFromJob throws', async () => {
    generateTaskFromJob.mockRejectedValueOnce(new Error('Prompt template broken'));
    await executeScheduledJob('job-fail');
    expect(recordJobGateSkip).toHaveBeenCalledWith('job-fail');
    expect(getScheduledActionReservations()).toBe(0);
  });

  it('calls recordJobGateSkip when executeScriptJob fails', async () => {
    globalThis.__nextJobType = 'script';
    executeScriptJob.mockRejectedValueOnce(new Error('Syntax error in job script'));
    await executeScheduledJob('script-fail');
    expect(recordJobGateSkip).toHaveBeenCalledWith('script-fail');
    expect(getScheduledActionReservations()).toBe(0);
  });

  it('calls recordJobGateSkip when executeShellJob fails', async () => {
    globalThis.__nextJobType = 'shell';
    executeShellJob.mockRejectedValueOnce(new Error('Command failed: exit 1'));
    await executeScheduledJob('shell-fail');
    expect(recordJobGateSkip).toHaveBeenCalledWith('shell-fail');
    expect(getScheduledActionReservations()).toBe(0);
  });
});

describe('registerSingleJobSchedule', () => {
  it('cancels a disabled job instead of registering a recurring timer', async () => {
    scheduleEvent.mockClear();
    cancelEvent.mockClear();
    getJob.mockResolvedValueOnce({ id: 'job-disabled', enabled: false });

    await registerSingleJobSchedule('job-disabled');

    expect(cancelEvent).toHaveBeenCalledWith('job:job-disabled');
    expect(scheduleEvent).not.toHaveBeenCalled();
  });

  // #6375 — this is the scheduler that actually arms the timer. An on-demand
  // job stays enabled (so POST /jobs/:id/trigger still runs it) but must never
  // be given a fire time; before the cadence existed, a null intervalMs reached
  // `lastRun + intervalMs` and produced a bogus delay instead.
  it('arms no timer for an enabled on-demand job, and cancels any stale one', async () => {
    scheduleEvent.mockClear();
    cancelEvent.mockClear();
    getJob.mockResolvedValueOnce({
      id: 'job-on-demand',
      name: 'Manual only',
      enabled: true,
      interval: 'on-demand',
      intervalMs: null,
    });

    await registerSingleJobSchedule('job-on-demand');

    expect(scheduleEvent).not.toHaveBeenCalled();
    expect(cancelEvent).toHaveBeenCalledWith('job:job-on-demand');
  });

  it('a weekdaysOnly on-demand job with a stale scheduledTime still arms no timer', async () => {
    // The synthesized daily/weekday cron path keys on weekdaysOnly, so it would
    // otherwise capture an on-demand job that carried those fields over.
    scheduleEvent.mockClear();
    cancelEvent.mockClear();
    getJob.mockResolvedValueOnce({
      id: 'job-on-demand-weekday',
      name: 'Manual only',
      enabled: true,
      interval: 'on-demand',
      intervalMs: null,
      weekdaysOnly: true,
      scheduledTime: '04:30',
    });

    await registerSingleJobSchedule('job-on-demand-weekday');

    expect(scheduleEvent).not.toHaveBeenCalled();
  });

  it('still arms a timer for a recurring job', async () => {
    scheduleEvent.mockClear();
    getJob.mockResolvedValueOnce({
      id: 'job-daily',
      name: 'Daily',
      enabled: true,
      interval: 'daily',
      intervalMs: 86_400_000,
    });

    await registerSingleJobSchedule('job-daily');

    expect(scheduleEvent).toHaveBeenCalledWith(
      expect.objectContaining({ id: 'job:job-daily', type: 'once' })
    );
    expect(scheduleEvent.mock.calls[0][0].delayMs).toBeGreaterThan(0);
  });
});

it('wakes at a future app deadline even while the task is ready for another app', async () => {
  vi.useFakeTimers();
  vi.setSystemTime(new Date('2026-01-01T16:00:00Z'));
  try {
    getUpcomingTasks.mockResolvedValue([{ taskType: 'do-release', status: 'ready', eligibleIn: 0,
      nextScheduledAt: Date.now() + 300000 }]);
    await scheduleNextImprovementCheck();
    expect(scheduleEvent).toHaveBeenLastCalledWith(expect.objectContaining({
      id: 'cos-improvement-check', delayMs: 300000
    }));
  } finally {
    vi.useRealTimers();
  }
});

// #7527: getUpcomingTasks now REJECTS (rather than swallowing input failures
// into a misleadingly "complete" []) when app inventory/override/readiness
// reads fail. cosJobScheduler owns that failure — it must retry soon instead
// of falling back to the 1h default, which could sleep through a cron minute.
describe('scheduleNextImprovementCheck — getUpcomingTasks input failure (#7527)', () => {
  beforeEach(async () => {
    // Reset the module-level "input unavailable" transition flag to a known
    // state (false) before each test, so the log-once assertions below don't
    // depend on execution order with other tests in this file.
    getScheduledEvent.mockReturnValue(null);
    getUpcomingTasks.mockResolvedValueOnce([]);
    await scheduleNextImprovementCheck();
  });

  afterEach(() => {
    vi.useRealTimers();
  });

  it('arms a short retry instead of the 1h fallback, and logs the outage once (not per consecutive failure)', async () => {
    vi.useFakeTimers();
    vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));
    scheduleEvent.mockClear();
    const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});

    getUpcomingTasks.mockRejectedValueOnce(new Error('inventory read failed'));
    await scheduleNextImprovementCheck();
    expect(scheduleEvent).toHaveBeenLastCalledWith(expect.objectContaining({
      id: 'cos-improvement-check', delayMs: 15000
    }));
    expect(errSpy).toHaveBeenCalledTimes(1);

    getUpcomingTasks.mockRejectedValueOnce(new Error('still down'));
    await scheduleNextImprovementCheck();
    expect(errSpy).toHaveBeenCalledTimes(1);

    errSpy.mockRestore();
  });

  it('logs recovery exactly once when getUpcomingTasks succeeds again', async () => {
    vi.useFakeTimers();
    vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));
    vi.spyOn(console, 'error').mockImplementation(() => {});
    getUpcomingTasks.mockRejectedValueOnce(new Error('inventory read failed'));
    await scheduleNextImprovementCheck();

    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
    getUpcomingTasks.mockResolvedValueOnce([]);
    await scheduleNextImprovementCheck();
    expect(logSpy).toHaveBeenCalledTimes(1);

    logSpy.mockRestore();
  });

  it('preserves an already-armed earlier deadline instead of pushing it out to the 15s retry', async () => {
    vi.useFakeTimers();
    vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));
    scheduleEvent.mockClear();
    vi.spyOn(console, 'error').mockImplementation(() => {});
    getScheduledEvent.mockReturnValue({ active: true, nextRunAt: Date.now() + 5000 });

    getUpcomingTasks.mockRejectedValueOnce(new Error('inventory read failed'));
    await scheduleNextImprovementCheck();

    expect(scheduleEvent).toHaveBeenLastCalledWith(expect.objectContaining({
      id: 'cos-improvement-check', delayMs: 5000
    }));
  });

  it('re-arms via the retry handler even when its own finally-block recompute also fails (no orphaned one-shot)', async () => {
    // This is the trap called out in #7527: the fired handler recomputes the
    // next arm from its own `finally`. If that recompute threw instead of
    // catching internally, the eventScheduler's 'once' rearm logic would
    // deactivate this fired event with no replacement — the improvement
    // cadence would die silently until a process restart.
    vi.useFakeTimers();
    vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));
    scheduleEvent.mockClear();
    getScheduledEvent.mockReturnValue(null);
    vi.spyOn(console, 'error').mockImplementation(() => {});
    canQueueImprovementTasks.mockReturnValue(false);

    getUpcomingTasks.mockRejectedValueOnce(new Error('inventory read failed'));
    await scheduleNextImprovementCheck();
    const retryHandler = scheduleEvent.mock.calls.at(-1)[0].handler;

    scheduleEvent.mockClear();
    getUpcomingTasks.mockRejectedValueOnce(new Error('still down'));
    await expect(retryHandler()).resolves.toBeUndefined();

    expect(scheduleEvent).toHaveBeenCalledTimes(1);
    expect(scheduleEvent).toHaveBeenLastCalledWith(expect.objectContaining({ id: 'cos-improvement-check' }));
  });
});

describe('handleJobSpawnFailed — no 1s re-fire loop after a failed dispatch', () => {
  beforeEach(() => {
    scheduleEvent.mockClear();
    recordJobGateSkip.mockClear();
    generateTaskFromJob.mockClear();
  });

  it('a held dispatch retries on the 60s defer and re-runs the job at most once per minute', async () => {
    await handleJobSpawnFailed('job-held', 'held');

    expect(scheduleEvent).toHaveBeenCalledTimes(1);
    const evt = scheduleEvent.mock.calls[0][0];
    expect(evt).toMatchObject({ id: 'job:job-held', type: 'once', delayMs: 60000 });
    expect(recordJobGateSkip).not.toHaveBeenCalled();

    await evt.handler();
    expect(generateTaskFromJob).toHaveBeenCalledTimes(1);
    clearSpawningJob('job-held');
  });

  it('a spawn error records a gate skip and re-arms via the normal schedule, not a 1s retry', async () => {
    getJob.mockResolvedValueOnce({ id: 'job-err', name: 'job-err', enabled: true, intervalMs: 86_400_000, lastRun: new Date().toISOString() });

    await handleJobSpawnFailed('job-err', 'failed');

    expect(recordJobGateSkip).toHaveBeenCalledWith('job-err');
    expect(scheduleEvent).toHaveBeenCalledTimes(1);
    expect(scheduleEvent.mock.calls[0][0].delayMs).toBeGreaterThan(60000);
  });
});