Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

refactor: split system-health warning rules and dismissal pruning out of getSystemHealthSnapshot (#12948)

[._.]/ Adam Eivy244ff7a5e013
diff --git a/server/services/systemHealthSnapshot.js b/server/services/systemHealthSnapshot.js
index 9c2b73f238..283be28748 100644
--- a/server/services/systemHealthSnapshot.js
+++ b/server/services/systemHealthSnapshot.js
@@ -80,42 +80,42 @@ function noteCosProbe(source, available, error) {
else console.error(`❌ CoS health probe unavailable (source=${source}, code=${probeErrorCode(error)})`);
}
-const patchHealth = (current, patch) => ({ ...current, health: { ...(current.health || {}), ...patch } });
-export async function getSystemHealthSnapshot() {
- const startTime = Date.now();
- const failedProbe = Symbol('failed health probe');
+const patchHealth = (current, patch) => ({ ...current, health: { ...(current.health || {}), ...patch } });
- // Gather data in parallel
- const [pm2Processes, appStatusSummary, cosStatus, cosPendingTaskIds, cosAgents, self, dbHealth, version, diskStats, dataDiskStats, filesystemDevices, memStats, healthSettings, forgeHealth, mediaCapacity, reviewerConfigHealth] = await Promise.all([
+const FAILED_PROBE = Symbol('failed health probe');
+
+const probeFailureWarning = (source, message) => ({
+ type: 'probe-unavailable', source, status: 'unavailable', severity: 'warning', message, dismissible: false
+});
+
+// Gather every probe in parallel. Per-probe error policy lives here: most
+// degrade to null or a fallback value, probes whose absence must be reported
+// resolve to FAILED_PROBE.
+function probeHealthInputs() {
+ const cosProbe = (source, read) => read().then(
+ (value) => { noteCosProbe(source, true); return value; },
+ (error) => { noteCosProbe(source, false, error); return FAILED_PROBE; }
+ );
+ return Promise.all([
listProcessesStrict().catch(() => null),
getAppStatusSummary().catch((error) => {
- const code = typeof error?.code === 'string' && /^[A-Z][A-Z0-9_]{0,31}$/.test(error.code) ? error.code : 'unknown';
- console.error(`❌ App health probe failed (operation=getAppStatusSummary, code=${code})`);
- return failedProbe;
+ console.error(`❌ App health probe failed (operation=getAppStatusSummary, code=${probeErrorCode(error)})`);
+ return FAILED_PROBE;
}),
- cos.getStatus().then(
- (status) => { noteCosProbe('cos-status', true); return status; },
- (error) => { noteCosProbe('cos-status', false, error); return failedProbe; }
- ),
+ cosProbe('cos-status', () => cos.getStatus()),
// Queue depth is read here rather than taken off `getStatus()`, which has no
// such field — `cosStatus.queueLength` never existed, so the widget's
// "N queued" was dead and always rendered 0. Both reads ride the same
// `loadState()`/parse caches `getStatus()` above already warmed.
- cos.getPendingTaskIds().then(
- (ids) => { noteCosProbe('cos-queue', true); return ids; },
- (error) => { noteCosProbe('cos-queue', false, error); return failedProbe; }
- ),
- cos.getAgents().then(
- (agents) => { noteCosProbe('cos-agents', true); return agents; },
- (error) => { noteCosProbe('cos-agents', false, error); return failedProbe; }
- ),
+ cosProbe('cos-queue', () => cos.getPendingTaskIds()),
+ cosProbe('cos-agents', () => cos.getAgents()),
getSelf().catch(() => null),
checkHealth().catch(() => ({ connected: false, hasSchema: false, error: 'Health check failed' })),
getCurrentVersion().catch(() => null),
statfs('/').catch((error) => {
console.error(`❌ Root filesystem health probe failed: ${error.message}`);
- return failedProbe;
+ return FAILED_PROBE;
}),
statfs(PATHS.data).catch(() => null),
Promise.all(['/', PATHS.data].map(path => stat(path).then(result => result.dev).catch(() => null))),
@@ -128,48 +128,28 @@ export async function getSystemHealthSnapshot() {
import('../services/codeReview.js')
.then(({ getReviewerConfigHealth }) => getReviewerConfigHealth())
.catch(() => ({ status: 'unknown', configFaults: {} }))
- ]);
- const { thresholds, dismissedWarnings, thresholdsAvailable } = healthSettings;
-
- const memUsagePercent = Math.round((memStats.used / memStats.total) * 100);
- const cpuLoad = os.loadavg()[0]; // 1-minute load average
- const cpuCount = os.cpus().length;
- const cpuUsagePercent = Math.round((cpuLoad / cpuCount) * 100);
-
- // Disk usage (root filesystem).
- // bavail = blocks available to unprivileged users (what the user can actually fill).
- // Derive used/usagePercent from the same figure so `used + free === total` and
- // the UI's percent corresponds to the displayed `free`.
- const parsedDisk = diskStats === failedProbe ? null : parseFilesystemStats(diskStats);
- const disk = parsedDisk && {
- total: parsedDisk.total,
- used: parsedDisk.used,
- free: parsedDisk.free,
- usagePercent: parsedDisk.usagePercent,
- };
-
- const dataDisk = parseFilesystemStats(dataDiskStats);
- const sameFilesystem = filesystemDevices.every(device => device != null)
- ? filesystemDevices[0] === filesystemDevices[1] : null;
+ ]).then(([pm2Processes, appStatusSummary, cosStatus, cosPendingTaskIds, cosAgents, self, dbHealth, version, diskStats, dataDiskStats, filesystemDevices, memStats, healthSettings, forgeHealth, mediaCapacity, reviewerConfigHealth]) => ({
+ pm2Processes, appStatusSummary, cosStatus, cosPendingTaskIds, cosAgents, self, dbHealth, version, diskStats, dataDiskStats, filesystemDevices, memStats, healthSettings, forgeHealth, mediaCapacity, reviewerConfigHealth
+ }));
+}
- // Process status summary from PM2. Processes whose exit is expected (a desktop
- // app the user closed) are excluded from the FAILURE-bearing counts: a quit game
- // window would otherwise force overallHealth to 'critical' below and light up
- // the dashboard widget and the OpenWorld HUD until the PM2 entry is manually
- // cleared. `expectedExit` describes a process's exit *semantics*, not its
- // liveness — so `online` counts every process, exempt or not, or a *running*
- // desktop app would sit in `total` and in no status bucket at all and the
- // dashboard would read "5/6 · all running" with all six up. Resource totals
- // likewise cover every process. See issue #2991.
- // When PM2 read fails, pm2Processes is null and we set processStats to null
- // to signal unavailable rather than zero.
- let processStats = null;
- let annotated = [];
- let supervised = [];
- if (pm2Processes !== null) {
- annotated = await annotateExpectedExit(pm2Processes);
- supervised = annotated.filter(p => !p.expectedExit);
- processStats = {
+// Process status summary from PM2. Processes whose exit is expected (a desktop
+// app the user closed) are excluded from the FAILURE-bearing counts: a quit game
+// window would otherwise force overallHealth to 'critical' and light up the
+// dashboard widget and the OpenWorld HUD until the PM2 entry is manually
+// cleared. `expectedExit` describes a process's exit *semantics*, not its
+// liveness — so `online` counts every process, exempt or not, or a *running*
+// desktop app would sit in `total` and in no status bucket at all. Resource
+// totals likewise cover every process. See issue #2991.
+// A failed PM2 read yields processStats null (unavailable), not zeros.
+async function deriveProcessStats(pm2Processes) {
+ if (pm2Processes === null) return { processStats: null, supervised: [] };
+ const annotated = await annotateExpectedExit(pm2Processes);
+ const supervised = annotated.filter(p => !p.expectedExit);
+ const sum = (list, key) => list.reduce((total, p) => total + (p[key] || 0), 0);
+ return {
+ supervised,
+ processStats: {
total: pm2Processes.length,
online: annotated.filter(p => p.status === 'online').length,
stopped: supervised.filter(p => p.status === 'stopped').length,
@@ -179,206 +159,206 @@ export async function getSystemHealthSnapshot() {
desktopExited: annotated.filter(
p => p.expectedExit && ['errored', 'stopped'].includes(p.status)
).length,
- totalMemory: pm2Processes.reduce((sum, p) => sum + (p.memory || 0), 0),
- totalCpu: pm2Processes.reduce((sum, p) => sum + (p.cpu || 0), 0),
- totalRestarts: pm2Processes.reduce((sum, p) => sum + (p.restarts || 0), 0),
- unstableRestarts: supervised.reduce((sum, p) => sum + (p.unstableRestarts || 0), 0)
- };
- }
-
- // App status summary — PM2-managed apps only (Xcode/iOS-native projects
- // have no detectable runtime state, so they're tracked under `unmanaged`
- // and excluded from the running denominator)
- // Numeric placeholders preserve older clients' shape, not measurements.
- const appStats = appStatusSummary === failedProbe
- ? { total: 0, online: 0, stopped: 0, notStarted: 0, unknown: 0, unmanaged: 0, degraded: true, status: 'unavailable' }
- : appStatusSummary;
-
- // Determine overall health status. Each condition below records its
- // severity on the warning itself rather than mutating `overallHealth`
- // inline, because a dismissed warning (see loadDismissedWarnings above)
- // must not count toward the badge — overallHealth is derived once, after
- // dismissals are filtered out, from whatever warnings remain visible.
- const rawWarnings = [];
-
- // Memory occupancy and CPU load describe work, not a health failure.
-
- if (!thresholdsAvailable) {
- rawWarnings.push({
- type: 'health-settings',
- severity: 'warning',
- message: 'System health settings are unavailable; default thresholds are being used and saved warning dismissals were ignored.',
- dismissible: false
- });
- }
+ totalMemory: sum(pm2Processes, 'memory'),
+ totalCpu: sum(pm2Processes, 'cpu'),
+ totalRestarts: sum(pm2Processes, 'restarts'),
+ unstableRestarts: sum(supervised, 'unstableRestarts')
+ }
+ };
+}
- if (diskStats === failedProbe) {
- rawWarnings.push({ type: 'probe-unavailable', source: 'disk', status: 'unavailable', severity: 'warning', message: 'Disk status unavailable', dismissible: false });
+function diskThresholdWarning(type, labelPrefix, usagePercent, thresholds) {
+ if (usagePercent >= thresholds.diskCritical) {
+ return { type, severity: 'critical', message: `${labelPrefix} usage at or above ${thresholds.diskCritical}%` };
}
- if (!dataDisk) {
- rawWarnings.push({ type: 'probe-unavailable', source: 'data-disk', status: 'unavailable', severity: 'warning', message: 'Runtime data disk status unavailable', dismissible: false });
- }
- if (cosStatus === failedProbe) {
- rawWarnings.push({ type: 'probe-unavailable', source: 'cos', status: 'unavailable', severity: 'warning', message: 'Chief of Staff status unavailable', dismissible: false });
+ if (usagePercent >= thresholds.diskWarn) {
+ return { type, severity: 'warning', message: `${labelPrefix} usage at or above ${thresholds.diskWarn}%` };
}
+ return null;
+}
+
+function cosProbeWarnings({ cosStatus, cosPendingTaskIds, cosAgents }) {
+ if (cosStatus === FAILED_PROBE) return [probeFailureWarning('cos', 'Chief of Staff status unavailable')];
// Queue and agent reads fail independently of getStatus(); only meaningful
// while the daemon status itself was readable (otherwise 'cos' already warns).
- if (cosStatus !== failedProbe) {
- if (cosPendingTaskIds === failedProbe) {
- rawWarnings.push({ type: 'probe-unavailable', source: 'cos-queue', status: 'unavailable', severity: 'warning', message: 'Chief of Staff queue unavailable', dismissible: false });
- }
- if (cosAgents === failedProbe) {
- rawWarnings.push({ type: 'probe-unavailable', source: 'cos-agents', status: 'unavailable', severity: 'warning', message: 'Chief of Staff agent census unavailable', dismissible: false });
- }
- }
- if (pm2Processes === null) {
- rawWarnings.push({ type: 'probe-unavailable', source: 'pm2', status: 'unavailable', severity: 'warning', message: 'Process manager (PM2) status unavailable', dismissible: false });
- }
+ const warnings = [];
+ if (cosPendingTaskIds === FAILED_PROBE) warnings.push(probeFailureWarning('cos-queue', 'Chief of Staff queue unavailable'));
+ if (cosAgents === FAILED_PROBE) warnings.push(probeFailureWarning('cos-agents', 'Chief of Staff agent census unavailable'));
+ return warnings;
+}
+function diskWarnings({ disk, dataDisk, sameFilesystem, thresholds }) {
// Concurrent probes may sample a shared volume on either side of a threshold.
// Keep one legacy warning identity and use the more severe available reading.
const warningDisk = sameFilesystem === true && disk && dataDisk
? (disk.usagePercent >= dataDisk.usagePercent ? disk : dataDisk) : disk;
- if (warningDisk) {
- if (warningDisk.usagePercent >= thresholds.diskCritical) {
- rawWarnings.push({ type: 'disk', severity: 'critical', message: `Disk usage at or above ${thresholds.diskCritical}%` });
- } else if (warningDisk.usagePercent >= thresholds.diskWarn) {
- rawWarnings.push({ type: 'disk', severity: 'warning', message: `Disk usage at or above ${thresholds.diskWarn}%` });
- }
- }
-
+ const warnings = [];
+ if (warningDisk) warnings.push(diskThresholdWarning('disk', 'Disk', warningDisk.usagePercent, thresholds));
// Preserve the root warning identity and its saved dismissals on a shared volume.
if (dataDisk && !(sameFilesystem === true && disk)) {
- if (dataDisk.usagePercent >= thresholds.diskCritical) {
- rawWarnings.push({ type: 'data-disk', severity: 'critical', message: `Runtime data disk usage at or above ${thresholds.diskCritical}%` });
- } else if (dataDisk.usagePercent >= thresholds.diskWarn) {
- rawWarnings.push({ type: 'data-disk', severity: 'warning', message: `Runtime data disk usage at or above ${thresholds.diskWarn}%` });
- }
+ warnings.push(diskThresholdWarning('data-disk', 'Runtime data disk', dataDisk.usagePercent, thresholds));
}
+ return warnings.filter(Boolean);
+}
- if (processStats) {
- if (processStats.errored > 0) {
- rawWarnings.push({ type: 'process', severity: 'critical', message: `${processStats.errored} process(es) errored` });
- }
-
- if (processStats.unstableRestarts > 0) {
- const crashing = supervised.filter(p => (p.unstableRestarts || 0) > 0).map(p => p.name);
- const plural = processStats.unstableRestarts === 1 ? '' : 's';
- rawWarnings.push({
- type: 'restarts',
- severity: 'warning',
- message: `${processStats.unstableRestarts} crash-loop restart${plural} (${crashing.join(', ')})`
- });
- }
+function processWarnings(processStats, supervised) {
+ if (!processStats) return [];
+ const warnings = [];
+ if (processStats.errored > 0) {
+ warnings.push({ type: 'process', severity: 'critical', message: `${processStats.errored} process(es) errored` });
}
-
- // A degraded app summary means PM2 couldn't be read for one or more homes, so
- // those apps' online/stopped status is unknown — surface it rather than letting
- // the counts silently read as "everything not started."
- if (appStatusSummary === failedProbe) {
- rawWarnings.push({ type: 'probe-unavailable', source: 'apps', status: 'unavailable', severity: 'warning', message: 'Apps unavailable', dismissible: false });
- } else if (appStats.degraded) {
- const unknown = appStats.unknown || 0;
- rawWarnings.push({ type: 'apps', severity: 'warning', message: `App status unavailable for ${unknown} app(s) — PM2 read failed` });
+ if (processStats.unstableRestarts > 0) {
+ const crashing = supervised.filter(p => (p.unstableRestarts || 0) > 0).map(p => p.name);
+ const plural = processStats.unstableRestarts === 1 ? '' : 's';
+ warnings.push({
+ type: 'restarts',
+ severity: 'warning',
+ message: `${processStats.unstableRestarts} crash-loop restart${plural} (${crashing.join(', ')})`
+ });
}
+ return warnings;
+}
+function databaseWarning(dbHealth) {
if (!dbHealth.connected) {
- rawWarnings.push({ type: 'database', severity: 'warning', message: `PostgreSQL disconnected${dbHealth.error ? `: ${dbHealth.error}` : ''}` });
- } else if (!dbHealth.hasSchema) {
- rawWarnings.push({ type: 'database', severity: 'warning', message: 'PostgreSQL connected but schema missing' });
- } else if (dbHealth.hasCatalogSchema === false) {
- rawWarnings.push({ type: 'database', severity: 'warning', message: 'PostgreSQL connected and schema available but Catalog tables missing — open Settings → Database to troubleshoot' });
+ return { type: 'database', severity: 'warning', message: `PostgreSQL disconnected${dbHealth.error ? `: ${dbHealth.error}` : ''}` };
}
-
- // A `gh` that cannot reach the forge does not fail loudly anywhere else: the
- // call sites that read pull requests and issues swallow the error into an
- // empty list, so a blocked or unauthenticated CLI looks exactly like a repo
- // with nothing open — and an agent asked to file a PR reports success having
- // filed none. Warn only when gh is present but unusable; an install that
- // never had gh has opted out of those features rather than broken them.
- if (!forgeHealth.ok && forgeHealth.status !== 'not-installed') {
- rawWarnings.push({
- type: 'forge',
- severity: 'warning',
- message: `GitHub CLI unusable (${forgeHealth.status})${forgeHealth.remedy ? ` — ${forgeHealth.remedy}` : ''}`
- });
+ if (!dbHealth.hasSchema) {
+ return { type: 'database', severity: 'warning', message: 'PostgreSQL connected but schema missing' };
}
-
- const reviewerConfigFaults = reviewerConfigHealth.configFaults || {};
- if (Object.keys(reviewerConfigFaults).length) {
- rawWarnings.push({
- type: 'code-review',
- severity: 'warning',
- message: `Code review configuration needs attention for ${Object.keys(reviewerConfigFaults).join(', ')} — open Settings → Code Reviewers`
- });
+ if (dbHealth.hasCatalogSchema === false) {
+ return { type: 'database', severity: 'warning', message: 'PostgreSQL connected and schema available but Catalog tables missing — open Settings → Database to troubleshoot' };
}
+ return null;
+}
+
+// A degraded app summary means PM2 couldn't be read for one or more homes, so
+// those apps' online/stopped status is unknown — surface it rather than letting
+// the counts silently read as "everything not started."
+function appsWarning(appStatusSummary, appStats) {
+ if (appStatusSummary === FAILED_PROBE) return probeFailureWarning('apps', 'Apps unavailable');
+ if (!appStats.degraded) return null;
+ return { type: 'apps', severity: 'warning', message: `App status unavailable for ${appStats.unknown || 0} app(s) — PM2 read failed` };
+}
+
+// A `gh` that cannot reach the forge does not fail loudly anywhere else: the
+// call sites that read pull requests and issues swallow the error into an
+// empty list, so a blocked or unauthenticated CLI looks exactly like a repo
+// with nothing open — and an agent asked to file a PR reports success having
+// filed none. Warn only when gh is present but unusable; an install that
+// never had gh has opted out of those features rather than broken them.
+function forgeWarning(forgeHealth) {
+ if (forgeHealth.ok || forgeHealth.status === 'not-installed') return null;
+ return {
+ type: 'forge',
+ severity: 'warning',
+ message: `GitHub CLI unusable (${forgeHealth.status})${forgeHealth.remedy ? ` — ${forgeHealth.remedy}` : ''}`
+ };
+}
+
+function reviewerConfigWarning(reviewerConfigHealth) {
+ const faulty = Object.keys(reviewerConfigHealth.configFaults || {});
+ if (!faulty.length) return null;
+ return {
+ type: 'code-review',
+ severity: 'warning',
+ message: `Code review configuration needs attention for ${faulty.join(', ')} — open Settings → Code Reviewers`
+ };
+}
- if (!thresholdsAvailable) rawWarnings.forEach((warning) => { warning.dismissible = false; });
+// Pure rule list: every condition records its severity on the warning itself
+// rather than mutating overallHealth, because a dismissed warning must not
+// count toward the badge — overallHealth derives from what remains visible.
+// Memory occupancy and CPU load describe work, not a health failure.
+function buildHealthWarnings(inputs) {
+ const { thresholdsAvailable, diskStats, dataDisk, pm2Processes, processStats, supervised, appStatusSummary, appStats, dbHealth, forgeHealth, reviewerConfigHealth } = inputs;
+ const warnings = [
+ !thresholdsAvailable && {
+ type: 'health-settings',
+ severity: 'warning',
+ message: 'System health settings are unavailable; default thresholds are being used and saved warning dismissals were ignored.',
+ dismissible: false
+ },
+ diskStats === FAILED_PROBE && probeFailureWarning('disk', 'Disk status unavailable'),
+ !dataDisk && probeFailureWarning('data-disk', 'Runtime data disk status unavailable'),
+ ...cosProbeWarnings(inputs),
+ pm2Processes === null && probeFailureWarning('pm2', 'Process manager (PM2) status unavailable'),
+ ...diskWarnings(inputs),
+ ...processWarnings(processStats, supervised),
+ appsWarning(appStatusSummary, appStats),
+ databaseWarning(dbHealth),
+ forgeWarning(forgeHealth),
+ reviewerConfigWarning(reviewerConfigHealth)
+ ].filter(Boolean);
+ if (!thresholdsAvailable) warnings.forEach((warning) => { warning.dismissible = false; });
+ return warnings;
+}
- // A dismissal only stays applied while the warning it was recorded against
- // is still current (same type AND same message) — see loadHealthSettings.
- // Anything else (the condition cleared, or recurred with a different
- // message) drops out of `dismissedWarnings` here so a genuinely new
- // occurrence is never silently hidden by a stale record.
- const nextDismissedWarnings = {};
+// A dismissal only stays applied while the warning it was recorded against
+// is still current (same type AND same message) — see loadHealthSettings.
+// Anything else (the condition cleared, or recurred with a different message)
+// is returned in `staleDismissals` so a genuinely new occurrence is never
+// silently hidden by a stale record. `pendingPrunes` defaults to the module map.
+function reconcileDismissals(rawWarnings, dismissedWarnings, thresholdsAvailable, pendingPrunes = pendingDismissalPrunes) {
+ const kept = new Set();
const warnings = [];
for (const warning of rawWarnings) {
const dismissal = dismissedWarnings[warning.type];
- const pendingPrune = pendingDismissalPrunes.get(warning.type);
- if (warning.dismissible !== false && dismissal?.message === warning.message && !sameDismissal(pendingPrune, dismissal)) {
- nextDismissedWarnings[warning.type] = dismissal;
+ if (warning.dismissible !== false && dismissal?.message === warning.message
+ && !sameDismissal(pendingPrunes.get(warning.type), dismissal)) {
+ kept.add(warning.type);
continue;
}
warnings.push(warning);
}
+ const staleDismissals = thresholdsAvailable
+ ? Object.entries(dismissedWarnings).filter(([type]) => !kept.has(type))
+ : [];
+ return { warnings, staleDismissals };
+}
- if (thresholdsAvailable) {
- for (const [type, pending] of pendingDismissalPrunes) {
- if (!Object.hasOwn(dismissedWarnings, type) || !sameDismissal(dismissedWarnings[type], pending)) {
+// Owns the pendingDismissalPrunes bookkeeping and the settings write; a failed
+// write keeps the entry pending so a matching recurrence stays visible.
+async function pruneStaleDismissals(staleDismissals, dismissedWarnings) {
+ for (const [type, pending] of pendingDismissalPrunes) {
+ if (!Object.hasOwn(dismissedWarnings, type) || !sameDismissal(dismissedWarnings[type], pending)) {
+ pendingDismissalPrunes.delete(type);
+ }
+ }
+ if (!staleDismissals.length) return;
+ for (const [type, dismissal] of staleDismissals) pendingDismissalPrunes.set(type, dismissal);
+ try {
+ const saved = await updateSettingsWith((current) => {
+ const next = { ...(current.health?.dismissedWarnings || {}) };
+ for (const [type, dismissal] of staleDismissals) {
+ if (sameDismissal(next[type], dismissal)) delete next[type];
+ }
+ return patchHealth(current, { dismissedWarnings: next });
+ });
+ for (const [type, dismissal] of staleDismissals) {
+ if (sameDismissal(pendingDismissalPrunes.get(type), dismissal)
+ && !sameDismissal(saved?.health?.dismissedWarnings?.[type], dismissal)) {
pendingDismissalPrunes.delete(type);
}
}
-
- const staleDismissals = Object.entries(dismissedWarnings)
- .filter(([type]) => !Object.hasOwn(nextDismissedWarnings, type));
- if (staleDismissals.length) {
- for (const [type, dismissal] of staleDismissals) pendingDismissalPrunes.set(type, dismissal);
- try {
- const saved = await updateSettingsWith((current) => {
- const next = { ...(current.health?.dismissedWarnings || {}) };
- for (const [type, dismissal] of staleDismissals) {
- if (sameDismissal(next[type], dismissal)) delete next[type];
- }
- return patchHealth(current, { dismissedWarnings: next });
- });
- for (const [type, dismissal] of staleDismissals) {
- if (sameDismissal(pendingDismissalPrunes.get(type), dismissal)
- && !sameDismissal(saved?.health?.dismissedWarnings?.[type], dismissal)) {
- pendingDismissalPrunes.delete(type);
- }
- }
- } catch (error) {
- for (const [type] of staleDismissals) {
- console.error(`❌ Failed to prune stale system health warning dismissal (type=${type}, error=${error?.code || 'write-failed'})`);
- }
- }
+ } catch (error) {
+ for (const [type] of staleDismissals) {
+ console.error(`❌ Failed to prune stale system health warning dismissal (type=${type}, error=${error?.code || 'write-failed'})`);
}
}
+}
- const overallHealth = warnings.some(w => w.severity === 'critical')
- ? 'critical'
- : warnings.length > 0 ? 'warning' : 'healthy';
-
- // CoS status. Active and queued BOTH come off the one agent read when it is
- // readable — mixing `getStatus()`'s tally with a separately-read queue lets the
- // two skew, which is the defect one layer down (services/activeProcessing.js).
- // A task a live agent already holds is active, not queued (lib/cosSpawnWindow.js).
- // An unreadable list degrades to null — unknown, which the widget hides —
- // rather than to a manufactured zero.
- const agentsReadable = cosAgents !== failedProbe;
- const queueReadable = cosPendingTaskIds !== failedProbe;
- const heldByRunningAgent = agentsReadable ? runningAgentsByTaskId(cosAgents) : null;
- const cosInfo = cosStatus && cosStatus !== failedProbe ? {
+// CoS status. Active and queued BOTH come off the one agent read when it is
+// readable — mixing `getStatus()`'s tally with a separately-read queue lets the
+// two skew, which is the defect one layer down (services/activeProcessing.js).
+// A task a live agent already holds is active, not queued (lib/cosSpawnWindow.js).
+// An unreadable list degrades to null — unknown, which the widget hides —
+// rather than to a manufactured zero.
+function buildCosInfo({ cosStatus, cosPendingTaskIds, cosAgents }) {
+ if (!cosStatus || cosStatus === FAILED_PROBE) return null;
+ const agentsReadable = cosAgents !== FAILED_PROBE;
+ const queueReadable = cosPendingTaskIds !== FAILED_PROBE;
+ return {
running: cosStatus.running,
paused: cosStatus.paused,
activeAgents: agentsReadable
@@ -386,13 +366,93 @@ export async function getSystemHealthSnapshot() {
: (cosStatus.activeAgents || 0),
// Without the agent census the spawn-window subtraction is impossible, so
// the count is unknown rather than an over-reported "reconciled" figure.
- queuedTasks: queueReadable && agentsReadable ? unclaimedTaskIds(cosPendingTaskIds, heldByRunningAgent).length : null
- } : null;
+ queuedTasks: queueReadable && agentsReadable
+ ? unclaimedTaskIds(cosPendingTaskIds, runningAgentsByTaskId(cosAgents)).length
+ : null
+ };
+}
+const withFormattedBytes = (fs) => fs && {
+ total: fs.total,
+ used: fs.used,
+ free: fs.free,
+ usagePercent: fs.usagePercent,
+ totalFormatted: formatBytes(fs.total),
+ usedFormatted: formatBytes(fs.used),
+ freeFormatted: formatBytes(fs.free)
+};
+
+function topProcesses(pm2Processes) {
+ if (pm2Processes === null) return null;
+ return [...pm2Processes]
+ .sort((a, b) => (b.memory || 0) - (a.memory || 0))
+ .slice(0, 10)
+ .map(p => ({
+ name: p.name,
+ status: p.status,
+ memory: p.memory || 0,
+ memoryFormatted: formatBytes(p.memory || 0),
+ cpu: p.cpu || 0,
+ restarts: p.restarts || 0,
+ unstableRestarts: p.unstableRestarts || 0
+ }));
+}
+
+function buildSystemSection({ memStats, disk, dataDisk, sameFilesystem }) {
+ const cpuLoad = os.loadavg()[0]; // 1-minute load average
+ const cpuCount = os.cpus().length;
const uptime = process.uptime();
- const uptimeFormatted = formatDuration(uptime * 1000);
+ return {
+ uptime,
+ uptimeFormatted: formatDuration(uptime * 1000),
+ memory: {
+ total: memStats.total,
+ used: memStats.used,
+ free: memStats.free,
+ usagePercent: Math.round((memStats.used / memStats.total) * 100),
+ totalFormatted: formatBytes(memStats.total),
+ usedFormatted: formatBytes(memStats.used),
+ freeFormatted: formatBytes(memStats.free)
+ },
+ cpu: {
+ cores: cpuCount,
+ loadAvg1m: cpuLoad,
+ usagePercent: Math.round((cpuLoad / cpuCount) * 100)
+ },
+ disk: withFormattedBytes(disk),
+ dataDisk: withFormattedBytes(dataDisk),
+ sameFilesystem
+ };
+}
- const _responseTime = Date.now() - startTime;
+export async function getSystemHealthSnapshot() {
+ const probes = await probeHealthInputs();
+ const { pm2Processes, appStatusSummary, self, dbHealth, version, diskStats, dataDiskStats, filesystemDevices, memStats, healthSettings, forgeHealth, mediaCapacity, reviewerConfigHealth } = probes;
+ const { thresholds, dismissedWarnings, thresholdsAvailable } = healthSettings;
+
+ // bavail = blocks available to unprivileged users (what the user can actually fill).
+ // Derive used/usagePercent from the same figure so `used + free === total` and
+ // the UI's percent corresponds to the displayed `free`.
+ const disk = diskStats === FAILED_PROBE ? null : parseFilesystemStats(diskStats);
+ const dataDisk = parseFilesystemStats(dataDiskStats);
+ const sameFilesystem = filesystemDevices.every(device => device != null)
+ ? filesystemDevices[0] === filesystemDevices[1] : null;
+
+ const { processStats, supervised } = await deriveProcessStats(pm2Processes);
+ // PM2-managed apps only (Xcode/iOS-native projects have no detectable runtime
+ // state, so they're `unmanaged`). Numeric placeholders preserve older
+ // clients' shape, not measurements.
+ const appStats = appStatusSummary === FAILED_PROBE
+ ? { total: 0, online: 0, stopped: 0, notStarted: 0, unknown: 0, unmanaged: 0, degraded: true, status: 'unavailable' }
+ : appStatusSummary;
+
+ const rawWarnings = buildHealthWarnings({ ...probes, ...healthSettings, disk, dataDisk, sameFilesystem, processStats, supervised, appStats });
+ const { warnings, staleDismissals } = reconcileDismissals(rawWarnings, dismissedWarnings, thresholdsAvailable);
+ if (thresholdsAvailable) await pruneStaleDismissals(staleDismissals, dismissedWarnings);
+
+ const overallHealth = warnings.some(w => w.severity === 'critical')
+ ? 'critical'
+ : warnings.length > 0 ? 'warning' : 'healthy';
return {
timestamp: new Date().toISOString(),
@@ -408,63 +468,23 @@ export async function getSystemHealthSnapshot() {
accepted: false,
},
warnings,
- system: {
- uptime,
- uptimeFormatted,
- memory: {
- total: memStats.total,
- used: memStats.used,
- free: memStats.free,
- usagePercent: memUsagePercent,
- totalFormatted: formatBytes(memStats.total),
- usedFormatted: formatBytes(memStats.used),
- freeFormatted: formatBytes(memStats.free)
- },
- cpu: {
- cores: cpuCount,
- loadAvg1m: cpuLoad,
- usagePercent: cpuUsagePercent
- },
- disk: disk ? {
- total: disk.total,
- used: disk.used,
- free: disk.free,
- usagePercent: disk.usagePercent,
- totalFormatted: formatBytes(disk.total),
- usedFormatted: formatBytes(disk.used),
- freeFormatted: formatBytes(disk.free)
- } : null,
- dataDisk: dataDisk ? {
- total: dataDisk.total,
- used: dataDisk.used,
- free: dataDisk.free,
- usagePercent: dataDisk.usagePercent,
- totalFormatted: formatBytes(dataDisk.total),
- usedFormatted: formatBytes(dataDisk.used),
- freeFormatted: formatBytes(dataDisk.free)
- } : null,
- sameFilesystem
- },
+ system: buildSystemSection({ memStats, disk, dataDisk, sameFilesystem }),
processes: processStats,
apps: appStats,
- cos: cosInfo,
+ cos: buildCosInfo(probes),
media: mediaCapacity,
database: dbHealth,
forge: forgeHealth,
codeReview: reviewerConfigHealth,
thresholds: thresholdsAvailable ? thresholds : undefined,
thresholdsAvailable,
- topProcesses: pm2Processes === null ? null : [...pm2Processes]
- .sort((a, b) => (b.memory || 0) - (a.memory || 0))
- .slice(0, 10)
- .map(p => ({
- name: p.name,
- status: p.status,
- memory: p.memory || 0,
- memoryFormatted: formatBytes(p.memory || 0),
- cpu: p.cpu || 0,
- restarts: p.restarts || 0,
- unstableRestarts: p.unstableRestarts || 0
- }))
+ topProcesses: topProcesses(pm2Processes)
};
}
+
+// Test hooks for the pure rule and dismissal steps.
+export {
+ buildHealthWarnings as _buildHealthWarnings,
+ reconcileDismissals as _reconcileDismissals,
+ pruneStaleDismissals as _pruneStaleDismissals
+};
diff --git a/server/services/systemHealthSnapshot.test.js b/server/services/systemHealthSnapshot.test.js
new file mode 100644
index 0000000000..46042d74a1
--- /dev/null
+++ b/server/services/systemHealthSnapshot.test.js
@@ -0,0 +1,112 @@
+import { describe, it, expect, vi } from 'vitest';
+
+// The pure rule/dismissal functions need none of the probes; stub the modules
+// so importing the file does not instantiate the CoS/DB/PM2 graph.
+vi.mock('./pm2.js', () => ({ listProcessesStrict: vi.fn() }));
+vi.mock('./appProcessStatus.js', () => ({ getAppStatusSummary: vi.fn(), annotateExpectedExit: vi.fn() }));
+vi.mock('./cos.js', () => ({}));
+vi.mock('./instanceIdentity.js', () => ({ getSelf: vi.fn() }));
+vi.mock('../lib/db.js', () => ({ checkHealth: vi.fn() }));
+vi.mock('./updateChecker.js', () => ({ getCurrentVersion: vi.fn() }));
+vi.mock('../lib/memoryStats.js', () => ({ getMemoryStats: vi.fn() }));
+vi.mock('./github.js', () => ({ checkGhHealth: vi.fn() }));
+vi.mock('./mediaCapacity.js', () => ({ getMediaCapacity: vi.fn() }));
+const updateSettingsWith = vi.hoisted(() => vi.fn());
+vi.mock('./settings.js', () => ({ getSettingsWithStatus: vi.fn(), updateSettingsWith }));
+
+const {
+ _buildHealthWarnings: buildHealthWarnings,
+ _reconcileDismissals: reconcileDismissals,
+ _pruneStaleDismissals: pruneStaleDismissals
+} = await import('./systemHealthSnapshot.js');
+
+const healthyInputs = (overrides = {}) => ({
+ thresholdsAvailable: true,
+ thresholds: { diskWarn: 90, diskCritical: 98 },
+ diskStats: {},
+ disk: { usagePercent: 10 },
+ dataDisk: { usagePercent: 10 },
+ sameFilesystem: false,
+ pm2Processes: [],
+ processStats: { errored: 0, unstableRestarts: 0 },
+ supervised: [],
+ appStatusSummary: {},
+ appStats: { degraded: false },
+ cosStatus: {},
+ cosPendingTaskIds: [],
+ cosAgents: [],
+ dbHealth: { connected: true, hasSchema: true },
+ forgeHealth: { ok: true },
+ reviewerConfigHealth: { configFaults: {} },
+ ...overrides
+});
+
+describe('buildHealthWarnings', () => {
+ it('emits nothing for a healthy snapshot', () => {
+ expect(buildHealthWarnings(healthyInputs())).toEqual([]);
+ });
+
+ it('reports unavailable probes as non-dismissible', () => {
+ const sample = buildHealthWarnings(healthyInputs({ pm2Processes: null, processStats: null, dataDisk: null }));
+ expect(sample.map(w => w.source)).toEqual(['data-disk', 'pm2']);
+ expect(sample.every(w => w.dismissible === false && w.type === 'probe-unavailable')).toBe(true);
+ });
+
+ it('uses one threshold ladder for root and data disks', () => {
+ const warnings = buildHealthWarnings(healthyInputs({
+ disk: { usagePercent: 91 }, dataDisk: { usagePercent: 99 }
+ }));
+ expect(warnings).toEqual([
+ { type: 'disk', severity: 'warning', message: 'Disk usage at or above 90%' },
+ { type: 'data-disk', severity: 'critical', message: 'Runtime data disk usage at or above 98%' }
+ ]);
+ });
+
+ it('keeps one warning identity on a shared volume and forces non-dismissible when settings are unavailable', () => {
+ const warnings = buildHealthWarnings(healthyInputs({
+ thresholdsAvailable: false, sameFilesystem: true,
+ disk: { usagePercent: 50 }, dataDisk: { usagePercent: 95 }
+ }));
+ expect(warnings.map(w => w.type)).toEqual(['health-settings', 'disk']);
+ expect(warnings.every(w => w.dismissible === false)).toBe(true);
+ });
+});
+
+describe('reconcileDismissals', () => {
+ const warning = { type: 'disk', severity: 'warning', message: 'Disk usage at or above 90%' };
+
+ it('suppresses a dismissal that is still current', () => {
+ const dismissed = { disk: { message: warning.message, dismissedAt: 't' } };
+ expect(reconcileDismissals([warning], dismissed, true, new Map()))
+ .toEqual({ warnings: [], staleDismissals: [] });
+ });
+
+ it('shows a changed message again and marks the old dismissal stale', () => {
+ const dismissed = { disk: { message: 'Disk usage at or above 80%', dismissedAt: 't' } };
+ const result = reconcileDismissals([warning], dismissed, true, new Map());
+ expect(result.warnings).toEqual([warning]);
+ expect(result.staleDismissals).toEqual([['disk', dismissed.disk]]);
+ });
+
+ it('keeps a warning visible while its dismissal awaits a prune', () => {
+ const dismissal = { message: warning.message, dismissedAt: 't' };
+ const result = reconcileDismissals([warning], { disk: dismissal }, true, new Map([['disk', { ...dismissal }]]));
+ expect(result.warnings).toEqual([warning]);
+ expect(result.staleDismissals).toEqual([['disk', dismissal]]);
+ });
+
+ it('never reports stale dismissals while settings are unavailable', () => {
+ expect(reconcileDismissals([], { disk: { message: 'x' } }, false, new Map()).staleDismissals).toEqual([]);
+ });
+});
+
+describe('pruneStaleDismissals', () => {
+ it('logs and swallows a failed settings write', async () => {
+ updateSettingsWith.mockRejectedValueOnce(Object.assign(new Error('boom'), { code: 'EIO' }));
+ const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
+ const dismissal = { message: 'm' };
+ await expect(pruneStaleDismissals([['gone', dismissal]], { gone: dismissal })).resolves.toBeUndefined();
+ expect(spy).toHaveBeenCalledWith(expect.stringContaining('type=gone, error=EIO'));
+ spy.mockRestore();
+ });
+});