Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

29.4 KB
Raw Blame
import { maintenance, isMaintenanceHold } from '../lib/maintenanceAdmission.js';
/**
 * CoS Job Scheduler Module
 *
 * The autonomous-job + improvement-check timer machinery extracted from cos.js.
 * Owns:
 *  - `computeNextJobFireTime` — cron/interval next-fire computation for a job.
 *  - `registerSingleJobSchedule` / `registerJobSchedules` / `unregisterJobSchedules`
 *    — per-job one-shot timers via the event scheduler.
 *  - `executeScheduledJob` — fires a due job (script/shell inline, or emits
 *    `task:ready` for an AI agent) with the spawning-guard machinery
 *    (`spawningJobIds` / `addSpawningJob` / `clearSpawningJob`) that prevents
 *    duplicate spawns when timers overlap.
 *  - `scheduleNextImprovementCheck` — the improvement-check cadence timer that
 *    queues eligible improvement tasks and asks the scheduler to dequeue. Owns
 *    a short retry (`IMPROVEMENT_CHECK_RETRY_MS`) when getUpcomingTasks' inputs
 *    are unavailable, instead of falling back to the 1h default (#7527).
 *
 * Self-contained — imports only sibling services (no import back to cos.js).
 * The paused check reads loadState() directly (cos.js's isPaused lived there),
 * and the improvement-check timer asks for a dequeue via the `cos:dequeue-requested`
 * event instead of calling cos.js's dequeueNextTask, so the spawn-side scheduler
 * stays in cos.js. cos.js's init() wires that event and the job lifecycle
 * (job:spawned/job:spawn-failed → clearSpawningJob + re-register) back to these.
 */

import { schedule as scheduleEvent, cancel as cancelEvent, getEvent as getScheduledEvent, parseCronToNextRun, parseRecurrenceToNextRun } from './eventScheduler.js';
import { getLocalParts, nextLocalTime } from '../lib/timezone.js';
import { getUserTimezone } from './userTimezone.js';
import { formatDuration } from '../lib/fileUtils.js';
import { loadState, isDaemonRunning, canQueueImprovementTasks } from './cosState.js';
import { getDomainMode } from '../lib/domainAutonomy.js';
import { isOnDemandJob } from '../lib/autonomousJobIntervals.js';
import { remainingActionBudget } from '../lib/domainBudgets.js';
import { getDomainBudgetStatus, recordDomainUsage } from './domainUsage.js';
import { cosEvents, emitLog } from './cosEvents.js';
import { getCosTasks } from './cosTaskStore.js';
import { queueEligibleImprovementTasks } from './cosTaskGenerator.js';
import { generateTaskFromJob, recordJobGateSkip, isScriptJob, executeScriptJob, isShellJob, executeShellJob } from './autonomousJobs.js';
import { checkJobGate, hasGate } from './jobGates.js';
import {
  acquireCosActionReservation,
  pendingCosActionReservations,
} from './cosAdmissionReservations.js';

/**
 * Compute the next fire time for an autonomous job.
 * Supports two scheduling modes:
 * 1. Cron mode: job.cronExpression defines the full schedule
 * 2. Interval mode: job.intervalMs + optional job.scheduledTime (HH:MM in user timezone)
 *
 * 3. On-demand: no cadence at all — returns null so no timer is armed.
 *
 * @param {Object} job - The job object
 * @param {string} timezone - IANA timezone string for interpreting scheduledTime/cron
 * @returns {number|null} Timestamp (ms) of next fire time, or null when the job never fires on a clock
 */
function computeNextJobFireTime(job, timezone) {
  // Convert scheduledTime (HH:MM) + interval to a cron expression so parseCronToNextRun
  // handles all "next occurrence after lastRun" logic without drift issues.
  // Only synthesize a daily/weekday cron for strictly daily or weekdaysOnly jobs.
  // Weekly/biweekly/sub-daily interval jobs fall through to the interval path below,
  // which correctly computes lastRun + intervalMs without scheduling them every day.
  // e.g. { interval: 'daily', scheduledTime: '04:30' } → '30 4 * * *'
  let cronExpr = job.cronExpression;
  if (job.cronSchedule) {
    const from = job.lastRun ? new Date(job.lastRun) : new Date();
    const next = parseRecurrenceToNextRun(job.cronSchedule, from, timezone);
    if (!next) {
      throw new Error(`Invalid calendar recurrence for autonomous job${job.id ? ` "${job.id}"` : ''}`);
    }
    return next.getTime();
  }
  // On-demand jobs never fire on a clock. Checked after the explicit cron
  // fields (a job switched to cron mode keeps whatever cadence it last had) but
  // before the synthesized daily/weekday cron below, which a weekdaysOnly
  // on-demand job would otherwise fall into.
  if (!cronExpr && isOnDemandJob(job)) return null;

  const isDailyCronCandidate = job.interval === 'daily' || job.weekdaysOnly;
  if (!cronExpr && job.scheduledTime && isDailyCronCandidate) {
    const match = String(job.scheduledTime).match(/^([01]\d|2[0-3]):([0-5]\d)$/);
    if (match) {
      const dayField = job.weekdaysOnly ? '1-5' : '*';
      cronExpr = `${Number(match[2])} ${Number(match[1])} * * ${dayField}`;
    }
  }

  if (cronExpr) {
    const from = job.lastRun ? new Date(job.lastRun) : new Date();
    const next = parseCronToNextRun(cronExpr, from, timezone);
    if (!next) {
      throw new Error(
        `Invalid cron expression for autonomous job` +
        (job.id ? ` "${job.id}"` : '') +
        `: ${cronExpr}`
      );
    }
    return next.getTime();
  }

  // Interval fallback: lastRun + intervalMs, then align to scheduledTime if set
  const lastRun = job.lastRun ? new Date(job.lastRun).getTime() : 0;
  let nextDue = lastRun + job.intervalMs;

  // Restore scheduledTime alignment for interval-mode jobs (e.g. weekly at 00:00).
  // nextLocalTime advances nextDue to the next occurrence of HH:MM in the user's timezone,
  // preventing drift when a run occurs slightly late.
  if (job.scheduledTime) {
    const match = String(job.scheduledTime).match(/^([01]\d|2[0-3]):([0-5]\d)$/);
    if (match) {
      const candidate = nextLocalTime(nextDue, Number(match[1]), Number(match[2]), timezone);
      if (candidate > nextDue) nextDue = candidate;
    }
  }

  if (job.weekdaysOnly) {
    const { dayOfWeek } = getLocalParts(new Date(nextDue), timezone);
    if (dayOfWeek === 0) nextDue += 24 * 60 * 60 * 1000; // Sunday → Monday
    if (dayOfWeek === 6) nextDue += 2 * 24 * 60 * 60 * 1000; // Saturday → Monday
  }

  return nextDue;
}

/**
 * Register a single autonomous job as a one-shot scheduled event.
 * After execution, re-registers for the next fire time.
 */
export async function registerSingleJobSchedule(jobId) {
  const { getJob } = await import('./autonomousJobs.js');
  const job = await getJob(jobId);
  if (!job || !job.enabled) {
    cancelEvent(`job:${jobId}`);
    return;
  }

  const timezone = await getUserTimezone();
  const nextFire = computeNextJobFireTime(job, timezone);
  // An on-demand job stays enabled (so POST /jobs/:id/trigger still runs it)
  // but arms no timer. Cancel any timer left over from a previous cadence.
  if (nextFire == null) {
    cancelEvent(`job:${jobId}`);
    return;
  }
  const delayMs = Math.max(nextFire - Date.now(), 1000);

  scheduleEvent({
    id: `job:${jobId}`,
    type: 'once',
    delayMs,
    handler: () => executeScheduledJob(jobId),
    metadata: { description: `Autonomous job: ${job.name}`, jobId }
  });
}

// Track jobs currently being spawned (between task:ready emit and agent registration)
// to prevent duplicate spawns when timers overlap or fire during spawn
const spawningJobIds = new Set();
const spawningJobTimeouts = new Map();

// Shared synchronous daily-action reservations (#984). The budget gate in
// executeScheduledJob reads recorded usage + spawningJobIds + running autonomous
// agents — but there are awaits between a job passing that gate and being
// counted by spawningJobIds (agent jobs) or recordDomainUsage (script/shell
// jobs). Two jobs coming due in the SAME event-loop tick could therefore both
// clear a `maxActionsPerDay == 1` gate before either was counted, nudging a soft
// daily cap by one. This counter is incremented the instant a job is admitted to
// fire — synchronously, before any further await — and included in the gate's
// in-flight term, so a same-tick sibling sees the reservation and is withheld.
// It's released the moment the job is handed off to spawningJobIds /
// recordDomainUsage or hits any exit path (a `finally` plus the agent handoff
// cover every path, so the two counters never double-count the same job).
// Test-only observability: lets the concurrency test assert the reservation
// counter settles back to zero after a tick (no leak across any exit path).
export function getScheduledActionReservations() {
  return pendingCosActionReservations();
}

function addSpawningJob(jobId) {
  spawningJobIds.add(jobId);
  // Auto-clear after 5 minutes if spawn never completes, and re-register the timer
  if (spawningJobTimeouts.has(jobId)) clearTimeout(spawningJobTimeouts.get(jobId));
  spawningJobTimeouts.set(jobId, setTimeout(() => {
    spawningJobIds.delete(jobId);
    spawningJobTimeouts.delete(jobId);
    emitLog('warn', `Job ${jobId} spawning timed out after 5m, re-registering schedule`, { jobId });
    registerSingleJobSchedule(jobId).catch(err =>
      console.error(`❌ Failed to re-register job schedule after spawn timeout for ${jobId}: ${err.message}`)
    );
  }, 5 * 60 * 1000));
}

/**
 * Whether this custom job already has a fire in flight — an agent registered and
 * running, or one handed to `spawningJobIds` whose agent has not registered yet.
 *
 * Exported because a quota burn asks the same question before queuing a job
 * (`quotaBurnInvoke.js`), and asking only the running-agent half of it — which
 * is all a caller outside this module can see, since `spawningJobIds` is
 * process-local — leaves the whole spawn window open to a double-queue.
 */
export function isJobFireInFlight(jobId, state) {
  if (spawningJobIds.has(jobId)) return true;
  return Object.values(state?.agents || {})
    .some((agent) => agent?.status === 'running' && agent?.metadata?.jobId === jobId);
}

export function clearSpawningJob(jobId) {
  spawningJobIds.delete(jobId);
  const timeout = spawningJobTimeouts.get(jobId);
  if (timeout) {
    clearTimeout(timeout);
    spawningJobTimeouts.delete(jobId);
  }
}

/**
 * Re-arm a job whose task:ready dispatch did not produce an agent. lastRun is
 * only advanced by job:spawned, so re-registering here would compute a past-due
 * fire time and re-fire every second.
 *  - 'held': a transient dispatch hold (runner down, capacity, maintenance...);
 *    keep the occurrence and retry on the 60s defer.
 *  - anything else (a real spawn error): record a skip so the job re-arms at its
 *    next real period, like a generation failure.
 */
export async function handleJobSpawnFailed(jobId, reason) {
  clearSpawningJob(jobId);
  if (reason === 'held') {
    emitLog('debug', `Job spawn held, retrying in 60s: ${jobId}`, { jobId });
    scheduleEvent({
      id: `job:${jobId}`,
      type: 'once',
      delayMs: 60000,
      handler: () => executeScheduledJob(jobId),
      metadata: { description: 'Autonomous job (dispatch held)', jobId }
    });
    return;
  }
  emitLog('warn', `Job spawn failed, re-arming at next period: ${jobId}`, { jobId });
  await recordJobGateSkip(jobId).catch(err =>
    console.error(`❌ Failed to record failure-skip for ${jobId}: ${err.message}`)
  );
  await registerSingleJobSchedule(jobId);
}

/**
 * Execute a scheduled autonomous job and re-register its timer.
 */
export async function executeScheduledJob(jobId) {
  if (maintenance.held()) {
    scheduleEvent({ id: `job:${jobId}`, type: 'once', delayMs: 60000, handler: () => executeScheduledJob(jobId), metadata: { jobId, description: 'Maintenance hold' } });
    return;
  }
  if (!isDaemonRunning()) return;

  const paused = (await loadState()).paused || false;
  if (paused) {
    // Re-register for later
    await registerSingleJobSchedule(jobId);
    return;
  }

  const { getJob } = await import('./autonomousJobs.js');
  const job = await getJob(jobId);
  if (!job || !job.enabled) return;

  const state = await loadState();
  if (!state.config.autonomousJobsEnabled) {
    // Re-register so it fires when re-enabled
    await registerSingleJobSchedule(jobId);
    return;
  }

  // Per-domain CoS auto-run gate: a SCHEDULED job firing automatically is an
  // autonomous action, so off/dry-run withhold it (dry-run logs what it would
  // have run). Manual triggers run through POST /jobs/:id/trigger (which calls
  // the spawn/execute helpers directly), not this scheduled path, so explicit
  // user intent is unaffected. Record a skip first (advances
  // lastRun without incrementing runCount, exactly like the gate-skip path) so
  // re-registration computes a FUTURE fire time — re-registering a past-due job
  // with stale lastRun would refire every second (the same 1s-loop the spawn
  // branch's comment warns about).
  let cosAutonomyMode = getDomainMode(state.config, 'cos');

  // Daily CoS budget (#711): a scheduled job firing is one autonomous action, so
  // once today's actions/minutes cap is hit, withhold it like `off` (and skip the
  // dry-run "would fire" log — over budget means stop, not narrate). The action
  // check counts in-flight autonomous runs too (recorded usage lags behind spawn),
  // so concurrent scheduled jobs can't slip past a small cap. Same gate-skip
  // bookkeeping below advances lastRun so re-registration is future-dated.
  const cosBudget = await getDomainBudgetStatus('cos');
  if (cosAutonomyMode !== 'off') {
    let overBudget = false;
    if (cosBudget.exceeded === 'minutes') {
      overBudget = true;
    } else if (cosBudget.budget?.maxActionsPerDay != null) {
      // In-flight = autonomous agents already running PLUS jobs admitted this
      // tick whose agent hasn't registered yet (`spawningJobIds`) PLUS jobs
      // admitted-to-fire this tick but not yet handed off to `spawningJobIds`
      // or recorded usage (the shared action reservations, #984). The reservation
      // term closes the same-tick window: two jobs coming due in one event-loop
      // tick used to each clear this gate before either was counted, because an
      // agent job reserves `spawningJobIds` only after later awaits and an inline
      // script/shell job records usage only post-execution. The reservation is
      // bumped synchronously the instant a job is admitted (below), so a same-tick
      // sibling reading this gate sees it and is correctly withheld.
      const runningAutonomous = Object.values(state.agents).filter(
        (a) => a.status === 'running' && a.metadata?.taskType && a.metadata.taskType !== 'user'
      ).length;
      const inFlight = runningAutonomous + spawningJobIds.size + pendingCosActionReservations();
      if (remainingActionBudget(cosBudget.budget, cosBudget.usage, inFlight) < 1) overBudget = true;
    }
    if (overBudget) {
      emitLog('info', `CoS auto-run paused — daily ${cosBudget.exceeded || 'actions'} budget reached, withholding scheduled job: ${job.name}`, { jobId, domainBudget: 'cos', exceeded: cosBudget.exceeded || 'actions' });
      cosAutonomyMode = 'off';
    }
  }

  if (cosAutonomyMode !== 'execute') {
    if (cosAutonomyMode === 'dry-run') {
      emitLog('info', `[dry-run] CoS auto-run would fire scheduled job: ${job.name}`, { jobId, domainAutonomy: 'cos' });
    }
    await recordJobGateSkip(jobId).catch(err =>
      console.error(`❌ Failed to record autonomy-skip for ${jobId}: ${err.message}`)
    );
    await registerSingleJobSchedule(jobId);
    return;
  }

  // Admitted to fire: reserve one daily action SYNCHRONOUSLY, before any further
  // await (#984). The budget gate above read the shared reservation count, and
  // there is no `await` between that read and this increment — so "read gate →
  // reserve" is an atomic critical section. Two jobs coming due in the same tick
  // serialize on it: the first reads remaining≥1 and reserves; the second reads
  // the reservation and is withheld. The reservation is handed off to
  // `spawningJobIds` (agent jobs) or recorded usage (script/shell jobs) the
  // moment either takes over the in-flight accounting, and the `finally`
  // releases it on every non-firing exit path (already-spawning, already-running,
  // capacity-defer, gate-skip, spawn failure). `releaseReservation` is idempotent
  // so the explicit handoff release and the `finally` can both fire harmlessly.
  const actionReservation = acquireCosActionReservation({
    budget: cosBudget.budget,
    usage: cosBudget.usage,
    inFlight: Object.values(state.agents).filter(
      (agent) => agent.status === 'running' && agent.metadata?.taskType && agent.metadata.taskType !== 'user'
    ).length + spawningJobIds.size,
    reservationId: `scheduled-job:${jobId}`,
  });
  if (!actionReservation.ok) {
    await recordJobGateSkip(jobId).catch((err) =>
      console.error(`❌ Failed to record budget-skip for ${jobId}: ${err.message}`)
    );
    await registerSingleJobSchedule(jobId);
    return;
  }
  const releaseReservation = actionReservation.release;

  try {
    // Script jobs and shell jobs execute directly without spawning an AI agent —
    // so they never reach completeAgent's budget tally. Record them against the CoS
    // daily budget (#711) here on success, since a fired scheduled job is exactly
    // the autonomous action the gate above withholds when over budget.
    if (isScriptJob(job)) {
      const startedAt = Date.now();
      const scriptOk = await executeScriptJob(job).then(() => true, err => {
        if (isMaintenanceHold(err)) return null;
        emitLog('error', `Script job failed: ${job.name} - ${err.message}`, { jobId: job.id });
        return false;
      });
      if (scriptOk === null) {
        scheduleEvent({ id: `job:${jobId}`, type: 'once', delayMs: 60000, handler: () => executeScheduledJob(jobId), metadata: { jobId } });
        return;
      }
      // Count the fired attempt against the budget whether it succeeded or failed —
      // a failing/looping scheduled job still consumes autonomous work + wall-clock,
      // and must not be able to bypass the daily cap.
      await recordDomainUsage('cos', { actions: 1, ms: Date.now() - startedAt })
        .catch(err => console.error(`❌ Failed to record CoS budget usage for script job ${job.id}: ${err.message}`));
      releaseReservation(); // recorded usage now owns the in-flight count for this action
      if (scriptOk) emitLog('info', `Script job executed: ${job.name}`, { jobId: job.id });
      else {
        await recordJobGateSkip(jobId).catch(err =>
          console.error(`❌ Failed to record failure-skip for ${jobId}: ${err.message}`)
        );
      }
    } else if (isShellJob(job)) {
      const startedAt = Date.now();
      const shellOk = await executeShellJob(job).then(() => true, err => {
        if (isMaintenanceHold(err)) return null;
        emitLog('error', `Shell job failed: ${job.name} - ${err.message}`, { jobId: job.id });
        return false;
      });
      if (shellOk === null) {
        scheduleEvent({ id: `job:${jobId}`, type: 'once', delayMs: 60000, handler: () => executeScheduledJob(jobId), metadata: { jobId } });
        return;
      }
      await recordDomainUsage('cos', { actions: 1, ms: Date.now() - startedAt })
        .catch(err => console.error(`❌ Failed to record CoS budget usage for shell job ${job.id}: ${err.message}`));
      releaseReservation(); // recorded usage now owns the in-flight count for this action
      if (shellOk) emitLog('info', `Shell job executed: ${job.name}`, { jobId: job.id });
      else {
        await recordJobGateSkip(jobId).catch(err =>
          console.error(`❌ Failed to record failure-skip for ${jobId}: ${err.message}`)
        );
      }
    } else {
      // Check if this job is already being spawned or has a running agent.
      // Don't re-register the timer here — the job:spawned handler will do it
      // after recordJobExecution updates lastRun. Re-registering with stale
      // lastRun causes a 1-second re-fire loop.
      if (isJobFireInFlight(jobId, state)) {
        emitLog('debug', `Job ${job.name} skipped - a fire is already in flight`, { jobId });
        return;
      }

      // Check capacity before spawning an agent
      const runningAgents = Object.values(state.agents).filter(a => a.status === 'running').length;
      if (runningAgents >= state.config.maxConcurrentAgents) {
        emitLog('debug', `Job ${job.name} deferred - no agent slots`, { jobId });
        // Retry in 60s
        scheduleEvent({
          id: `job:${jobId}`,
          type: 'once',
          delayMs: 60000,
          handler: () => executeScheduledJob(jobId),
          metadata: { description: `Autonomous job: ${job.name} (retry)`, jobId }
        });
        return;
      }

      // Run gate check — skip LLM if precondition not met
      // Gate errors fail-open (run the job) to avoid silently dropping scheduled work
      let gateResult;
      try {
        gateResult = await checkJobGate(jobId);
      } catch (gateErr) {
        emitLog('warn', `Job ${job.name} gate error, failing open: ${gateErr?.message || gateErr}`, { jobId });
        gateResult = { shouldRun: true, reason: 'Gate error — failing open' };
      }
      if (!gateResult.shouldRun) {
        emitLog('debug', `Job ${job.name} gate skipped: ${gateResult.reason}`, { jobId, gate: gateResult });
        // Update lastRun so the job reschedules at its normal interval, but don't increment runCount
        await recordJobGateSkip(jobId).catch(err =>
          console.error(`❌ Failed to record gate-skip for ${jobId}: ${err.message}`)
        );
        await registerSingleJobSchedule(jobId);
        return;
      }
      if (hasGate(jobId)) {
        emitLog('info', `Job ${job.name} gate passed: ${gateResult.reason}`, { jobId, gate: gateResult });
      }

      // Mark as spawning before emitting task:ready to prevent races
      addSpawningJob(jobId);
      releaseReservation(); // spawningJobIds now owns the in-flight count for this action
      try {
        const task = await generateTaskFromJob(job);
        emitLog('info', `Autonomous job firing: ${job.name}`, { jobId, category: job.category });
        cosEvents.emit('task:ready', task);
        // Don't re-register timer here — lastRun hasn't been updated yet, so
        // computeNextJobFireTime would return a past-due time and the timer would
        // fire in 1s, creating a rapid re-fire loop. The job:spawned handler
        // re-registers after recordJobExecution updates lastRun.
        return;
      } catch (err) {
        clearSpawningJob(jobId);
        emitLog('error', `Failed to fire autonomous job: ${job.name} - ${err?.message || err}`, { jobId, category: job.category });
        await recordJobGateSkip(jobId).catch(skipErr =>
          console.error(`❌ Failed to record failure-skip for ${jobId}: ${skipErr.message}`)
        );
      }
    }

    // Re-register for next fire time (script/shell jobs, early returns, and error paths)
    await registerSingleJobSchedule(jobId);
  } finally {
    // Catch-all: release on every exit path that didn't hand the reservation off
    // (already-spawning / already-running / capacity-defer / gate-skip /
    // generation+spawn failure). Idempotent, so a path that already handed off is
    // a no-op here.
    releaseReservation();
  }
}

/**
 * Register all enabled autonomous jobs as individual one-shot scheduled events.
 */
export async function registerJobSchedules() {
  const { getEnabledJobs } = await import('./autonomousJobs.js');
  const jobs = await getEnabledJobs();

  for (const job of jobs) {
    await registerSingleJobSchedule(job.id);
  }

  if (jobs.length > 0) {
    emitLog('info', `📅 Registered ${jobs.length} autonomous job schedule(s)`);
  }
}

/**
 * Cancel all autonomous job scheduled events.
 */
export async function unregisterJobSchedules() {
  const { getAllJobs } = await import('./autonomousJobs.js');
  const jobs = await getAllJobs();

  for (const job of jobs) {
    cancelEvent(`job:${job.id}`);
  }
}

// Cap for the "nothing scheduled" fallback delay.
const MAX_CHECK_INTERVAL = 60 * 60 * 1000;
// Retry cadence when getUpcomingTasks' inputs (app inventory/override/
// readiness reads) are unavailable — short enough that a transient outage
// self-heals well inside a cron minute, per #7527.
const IMPROVEMENT_CHECK_RETRY_MS = 15 * 1000;

// Transition-based logging: log once when the input read starts failing and
// once when it recovers, not on every 15s retry — a persistent outage would
// otherwise flood the log every 15s.
let improvementCheckInputUnavailable = false;

/**
 * The improvement-check one-shot handler — shared by every arm of the
 * 'cos-improvement-check' event, whether the delay came from a normal
 * calculation or the input-unavailable retry below. Queues eligible
 * improvement work, then re-arms via scheduleNextImprovementCheck().
 *
 * This is a 'once' event — the eventScheduler does NOT auto-reschedule it,
 * so the handler must re-arm itself. Re-arm in `finally` so a throw in the
 * body (loadState / getCosTasks / queueEligibleImprovementTasks) can't
 * permanently halt the improvement cadence until a process restart.
 * scheduleNextImprovementCheck() never throws (its own getUpcomingTasks call
 * is caught internally) — a naive throw from here would otherwise leave this
 * fired 'once' event deactivated with no replacement registered.
 */
async function runImprovementCheckOnce() {
  if (!isDaemonRunning()) return;
  try {
    const paused = (await loadState()).paused || false;
    if (paused) return;

    const state = await loadState();
    // Gate on the CoS auto-run domain (see canQueueImprovementTasks): off/
    // dry-run are planning postures that withhold the COS-TASKS.md mutation.
    if (canQueueImprovementTasks(state)) {
      const cosTaskData = await getCosTasks();
      await queueEligibleImprovementTasks(state, cosTaskData);
      cosEvents.emit('cos:dequeue-requested');
    }
  } finally {
    await scheduleNextImprovementCheck();
  }
}

/**
 * Schedule a one-shot timer for the next due improvement task.
 * When it fires, queues eligible improvement tasks and re-schedules.
 *
 * Called from three places (cos.js boot, the 'schedule:changed' handler, and
 * this same re-arm from runImprovementCheckOnce's `finally`) — every one of
 * them must end up with exactly one 'cos-improvement-check' timer armed, so
 * this function never throws. When getUpcomingTasks' inputs are unavailable
 * it arms a short retry instead of propagating (#7527).
 */
export async function scheduleNextImprovementCheck() {
  if (!isDaemonRunning()) return;

  const taskSchedule = await import('./taskSchedule.js');
  const now = Date.now();
  let delayMs;
  let description;
  let upcoming;

  try {
    // Pull a wider list so a perpetually-"ready" weekly task can't mask an upcoming
    // cron boundary. We sort by status (ready first), so 1-element peeks always miss
    // the cron slot when anything else is ready.
    upcoming = await taskSchedule.getUpcomingTasks(50);
  } catch (err) {
    // An app inventory/override/readiness read failed — the calculation is
    // UNAVAILABLE, not "nothing scheduled". Falling back to the 1h default
    // here could sleep through a cron minute a healthy read would have
    // caught, so arm a short retry and let the next attempt recompute the
    // real deadline once the input recovers.
    if (!improvementCheckInputUnavailable) {
      improvementCheckInputUnavailable = true;
      console.error(`❌ Upcoming-tasks calculation unavailable, retrying in 15s: ${err?.message || err}`);
    }

    // Don't let the retry push out an earlier deadline this same event is
    // already armed for (e.g. a cron boundary a prior successful calculation
    // found) — arm the sooner of "now + retry delay" and that deadline.
    const existing = getScheduledEvent('cos-improvement-check');
    const existingDelay = existing?.active && Number.isFinite(existing.nextRunAt)
      ? existing.nextRunAt - now
      : null;
    delayMs = existingDelay !== null && existingDelay > 0
      ? Math.min(existingDelay, IMPROVEMENT_CHECK_RETRY_MS)
      : IMPROVEMENT_CHECK_RETRY_MS;

    scheduleEvent({
      id: 'cos-improvement-check',
      type: 'once',
      delayMs: Math.max(delayMs, 1000),
      handler: runImprovementCheckOnce,
      metadata: { description: 'Improvement check retry (upcoming-tasks unavailable)' }
    });
    return;
  }

  if (improvementCheckInputUnavailable) {
    improvementCheckInputUnavailable = false;
    console.log('✅ Upcoming-tasks calculation recovered');
  }

  // Default: check again in 1 hour if nothing scheduled
  // Cap at 1 hour as a fallback for work whose next boundary is not known;
  // getUpcomingTasks also includes per-app cron overrides.
  delayMs = MAX_CHECK_INTERVAL;
  description = 'Periodic improvement check (1h)';

  // Keep future app deadlines even when another app makes the same task ready.
  // Ready tasks don't gate the delay — they'll be queued on whatever the next check
  // ends up being. Cron tasks DO gate the delay, because their firing window is a
  // single minute; missing it pushes the next attempt out by a full period.
  const nextScheduled = upcoming
    .map(t => ({ ...t, eligibleIn: t.nextScheduledAt > now
      ? t.nextScheduledAt - now
      : t.status === 'scheduled' ? t.eligibleIn : 0 }))
    .filter(t => t.eligibleIn > 0)
    .sort((a, b) => a.eligibleIn - b.eligibleIn)[0];

  if (nextScheduled && nextScheduled.eligibleIn < MAX_CHECK_INTERVAL) {
    delayMs = nextScheduled.eligibleIn;
    description = `Next improvement: ${nextScheduled.taskType} in ${formatDuration(delayMs)}`;
  } else if (nextScheduled) {
    description = `Next improvement: ${nextScheduled.taskType} in ${nextScheduled.eligibleInFormatted} (capped at 1h)`;
  }

  scheduleEvent({
    id: 'cos-improvement-check',
    type: 'once',
    delayMs: Math.max(delayMs, 1000),
    handler: runImprovementCheckOnce,
    metadata: { description }
  });
}