Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.2 KB
Raw Blame
/**
 * CoS Health Monitor Module
 *
 * Daemon health checks extracted from cos.js. Inspects PM2 process state and
 * memory usage, auto-restarts errored processes, records the latest health
 * snapshot to CoS state, and emits health events for downstream consumers.
 */

import { restartApp, listProcessesStrict, clearJlistCache } from './pm2.js';
import { getMemoryStats } from '../lib/memoryStats.js';
import { loadState, saveState, withStateLock, isDaemonRunning } from './cosState.js';
import { cosEvents, emitLog } from './cosEvents.js';
import { annotateExpectedExit } from './appProcessStatus.js';

// A restart that PM2 accepted proves the command ran, not that the process
// stayed up. Verification is bounded: a few fresh reads, waiting only while PM2
// still reports a transitional state. Exported for tests to shrink the wait.
export const RESTART_VERIFY = { attempts: 3, delayMs: 2000 };
const TRANSITIONAL = new Set(['launching', 'waiting restart', 'stopping']);
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));

// Stable identity: pm_id distinguishes cluster instances sharing a name.
const processKey = (p) => (p.pm_id ?? null) !== null ? `id:${p.pm_id}` : `name:${p.name}`;

async function summarizePm2(processes) {
  const annotated = await annotateExpectedExit(processes);
  const supervised = annotated.filter(p => !p.expectedExit);
  const erroredProcesses = supervised.filter(p => p.status === 'errored');
  // Reported on its own rather than folded into `errored` (which would
  // degrade health for a normal user action) or dropped from the totals.
  // Counted from the expected-exit processes only, so it never overlaps
  // `errored`/`stopped` — those count supervised processes exclusively.
  const desktopExited = annotated.filter(
    p => p.expectedExit && ['errored', 'stopped'].includes(p.status)
  ).length;
  // `online` counts EVERY process, exempt or not: the exemption is about
  // exit semantics, not liveness, so a running desktop app must still report
  // as online.
  return {
    erroredProcesses,
    pm2: {
      total: processes.length,
      online: annotated.filter(p => p.status === 'online').length,
      errored: erroredProcesses.length,
      stopped: supervised.filter(p => p.status === 'stopped').length,
      desktopExited
    }
  };
}

/**
 * Observe whether restarted processes came back. Returns
 * { processes: fresh list | null, unconfirmed: [{ name, reason }] }.
 */
async function verifyRecovery(restarted) {
  const wanted = new Map(restarted.map(p => [processKey(p), p]));
  let fresh = null;
  for (let attempt = 0; attempt < RESTART_VERIFY.attempts; attempt++) {
    if (attempt > 0) await sleep(RESTART_VERIFY.delayMs);
    clearJlistCache();
    fresh = await listProcessesStrict();
    if (fresh === null) continue;
    // Wait only while a restarted process is still in a transitional state;
    // online, errored, stopped or missing are all settled answers.
    const byKey = new Map(fresh.map(p => [processKey(p), p]));
    if (![...wanted.keys()].some(key => TRANSITIONAL.has(byKey.get(key)?.status))) break;
  }
  if (fresh === null) {
    return { processes: null, unconfirmed: [...wanted.values()].map(p => ({ name: p.name, reason: 'verification read failed' })) };
  }
  const byKey = new Map(fresh.map(p => [processKey(p), p]));
  const unconfirmed = [...wanted.entries()]
    .filter(([key]) => byKey.get(key)?.status !== 'online')
    .map(([key, p]) => {
      const status = byKey.get(key)?.status;
      if (!status) return { name: p.name, reason: 'missing after restart' };
      return { name: p.name, reason: TRANSITIONAL.has(status) ? `still ${status}` : status };
    });
  return { processes: fresh, unconfirmed };
}

/**
 * Run a daemon health check: inspect PM2 processes and memory, auto-restart
 * errored processes, store the result, and emit health events.
 */
export async function runHealthCheck() {
  if (!isDaemonRunning()) return;

  const state = await loadState();
  const issues = [];
  const metrics = {
    timestamp: new Date().toISOString(),
    pm2: null,
    memory: null,
    ports: null
  };

  // Check PM2 processes. `listProcessesStrict()` returns `null` when the read
  // itself FAILED (vs `[]` for a successful read with no processes) — the
  // absent-vs-empty contract from issue #968, now the one non-test owner of raw
  // `pm2 jlist` execution/parsing alongside `autofixer/shared.js` (#8164). A
  // failed read must not be recorded as zero processes: that would suppress
  // repair (no errored processes found) and misreport health as clean.
  const pm2Processes = await listProcessesStrict();

  if (pm2Processes === null) {
    metrics.pm2 = null;
    issues.push({
      type: 'error',
      category: 'processes',
      message: 'PM2 process read failed — process health is unavailable, not necessarily clean'
    });
  } else {
    // A process whose stopping is a normal outcome (a desktop app the user
    // closed) must not be auto-restarted — that would reopen the window they
    // just closed, the same relaunch loop `autorestart: false` prevents,
    // arriving by another path. See issue #2991.
    const summary = await summarizePm2(pm2Processes);
    const erroredProcesses = summary.erroredProcesses;
    metrics.pm2 = summary.pm2;

    // Check for runaway processes (too many)
    if (pm2Processes.length > state.config.maxTotalProcesses) {
      issues.push({
        type: 'warning',
        category: 'processes',
        message: `High process count: ${pm2Processes.length} PM2 processes (limit: ${state.config.maxTotalProcesses})`
      });
    }

    // Check for errored processes and auto-restart them
    if (erroredProcesses.length > 0) {
      const names = erroredProcesses.map(p => p.name);
      emitLog('warn', `🔄 ${names.length} errored PM2 process(es) detected: ${names.join(', ')} — attempting restart`);

      const restartResults = await Promise.all(erroredProcesses.map(async (proc) => {
        const name = proc.name;
        const result = await restartApp(name).catch(error => ({ success: false, error }));
        const uncertain = result.error?.outcome === 'unknown';
        if (!result.success) emitLog('error', `❌ ${uncertain ? 'Restart outcome unknown for' : 'Failed to restart'} ${name}: ${result.error.message}`);
        return { proc, success: result.success, uncertain };
      }));

      const failedRestarts = restartResults.filter(r => !r.success && !r.uncertain);
      if (failedRestarts.length > 0) {
        issues.push({
          type: 'error',
          category: 'processes',
          message: `${failedRestarts.length} errored PM2 process(es) failed to auto-restart: ${failedRestarts.map(r => r.proc.name).join(', ')}`
        });
      }

      const uncertainRestarts = restartResults.filter(r => r.uncertain);
      if (uncertainRestarts.length > 0) {
        issues.push({
          type: 'error',
          category: 'processes',
          message: `PM2 restart outcome unknown after timeout: ${uncertainRestarts.map(r => r.proc.name).join(', ')}; the daemon may have accepted the restart`
        });
      }

      // Command acceptance is not recovery: observe process state afresh.
      const accepted = restartResults.filter(r => r.success).map(r => r.proc);
      if (accepted.length > 0) {
        const { processes: fresh, unconfirmed } = await verifyRecovery(accepted);
        const unconfirmedKeys = new Set(unconfirmed.map(u => u.name));
        for (const proc of accepted) {
          if (!unconfirmedKeys.has(proc.name)) emitLog('success', `✅ Auto-restarted errored process (observed online): ${proc.name}`);
        }
        if (unconfirmed.length > 0) {
          const detail = unconfirmed.map(u => `${u.name} (${u.reason})`).join(', ');
          emitLog('error', `❌ Restart accepted but recovery not confirmed: ${detail}`);
          issues.push({
            type: 'error',
            category: 'processes',
            message: `${unconfirmed.length} PM2 process(es) restarted but recovery is unconfirmed: ${detail}`
          });
        }
        // Report fresh post-restart metrics when the observation succeeded.
        if (fresh !== null) metrics.pm2 = (await summarizePm2(fresh)).pm2;
      }
    }
  }

  // Keep memory telemetry without treating expected occupancy as a failure.
  metrics.memory = await getMemoryStats();

  // Store health check result with lock to prevent race conditions
  await withStateLock(async () => {
    const freshState = await loadState();
    freshState.stats.lastHealthCheck = metrics.timestamp;
    freshState.stats.healthIssues = issues;
    await saveState(freshState);
  });

  cosEvents.emit('health:check', { metrics, issues });

  // If there are critical issues, emit for potential automated response
  if (issues.filter(i => i.type === 'error').length > 0) {
    cosEvents.emit('health:critical', issues.filter(i => i.type === 'error'));
  }

  return { metrics, issues };
}

/**
 * Get latest health status
 */
export async function getHealthStatus() {
  const state = await loadState();
  return {
    lastCheck: state.stats.lastHealthCheck,
    // Older snapshots can survive until the next daemon poll.
    issues: (state.stats.healthIssues || []).filter(issue => issue.category !== 'memory'
      && !(issue.category === 'processes' && issue.message?.startsWith('Auto-restarted ')))
  };
}