Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
refactor: split system-health warning rules and dismissal pruning out of getSystemHealthSnapshot (#12948)
[._.]/ Adam Eivy244ff7a5e013
diff --git a/server/services/systemHealthSnapshot.js b/server/services/systemHealthSnapshot.jsindex 9c2b73f238..283be28748 100644--- a/server/services/systemHealthSnapshot.js+++ b/server/services/systemHealthSnapshot.js@@ -80,42 +80,42 @@ function noteCosProbe(source, available, error) {else console.error(`❌ CoS health probe unavailable (source=${source}, code=${probeErrorCode(error)})`);}-const patchHealth = (current, patch) => ({ ...current, health: { ...(current.health || {}), ...patch } });-export async function getSystemHealthSnapshot() {- const startTime = Date.now();- const failedProbe = Symbol('failed health probe');+const patchHealth = (current, patch) => ({ ...current, health: { ...(current.health || {}), ...patch } });- // Gather data in parallel- const [pm2Processes, appStatusSummary, cosStatus, cosPendingTaskIds, cosAgents, self, dbHealth, version, diskStats, dataDiskStats, filesystemDevices, memStats, healthSettings, forgeHealth, mediaCapacity, reviewerConfigHealth] = await Promise.all([+const FAILED_PROBE = Symbol('failed health probe');++const probeFailureWarning = (source, message) => ({+ type: 'probe-unavailable', source, status: 'unavailable', severity: 'warning', message, dismissible: false+});++// Gather every probe in parallel. Per-probe error policy lives here: most+// degrade to null or a fallback value, probes whose absence must be reported+// resolve to FAILED_PROBE.+function probeHealthInputs() {+ const cosProbe = (source, read) => read().then(+ (value) => { noteCosProbe(source, true); return value; },+ (error) => { noteCosProbe(source, false, error); return FAILED_PROBE; }+ );+ return Promise.all([listProcessesStrict().catch(() => null),getAppStatusSummary().catch((error) => {- const code = typeof error?.code === 'string' && /^[A-Z][A-Z0-9_]{0,31}$/.test(error.code) ? error.code : 'unknown';- console.error(`❌ App health probe failed (operation=getAppStatusSummary, code=${code})`);- return failedProbe;+ console.error(`❌ App health probe failed (operation=getAppStatusSummary, code=${probeErrorCode(error)})`);+ return FAILED_PROBE;}),- cos.getStatus().then(- (status) => { noteCosProbe('cos-status', true); return status; },- (error) => { noteCosProbe('cos-status', false, error); return failedProbe; }- ),+ cosProbe('cos-status', () => cos.getStatus()),// Queue depth is read here rather than taken off `getStatus()`, which has no// such field — `cosStatus.queueLength` never existed, so the widget's// "N queued" was dead and always rendered 0. Both reads ride the same// `loadState()`/parse caches `getStatus()` above already warmed.- cos.getPendingTaskIds().then(- (ids) => { noteCosProbe('cos-queue', true); return ids; },- (error) => { noteCosProbe('cos-queue', false, error); return failedProbe; }- ),- cos.getAgents().then(- (agents) => { noteCosProbe('cos-agents', true); return agents; },- (error) => { noteCosProbe('cos-agents', false, error); return failedProbe; }- ),+ cosProbe('cos-queue', () => cos.getPendingTaskIds()),+ cosProbe('cos-agents', () => cos.getAgents()),getSelf().catch(() => null),checkHealth().catch(() => ({ connected: false, hasSchema: false, error: 'Health check failed' })),getCurrentVersion().catch(() => null),statfs('/').catch((error) => {console.error(`❌ Root filesystem health probe failed: ${error.message}`);- return failedProbe;+ return FAILED_PROBE;}),statfs(PATHS.data).catch(() => null),Promise.all(['/', PATHS.data].map(path => stat(path).then(result => result.dev).catch(() => null))),@@ -128,48 +128,28 @@ export async function getSystemHealthSnapshot() {import('../services/codeReview.js').then(({ getReviewerConfigHealth }) => getReviewerConfigHealth()).catch(() => ({ status: 'unknown', configFaults: {} }))- ]);- const { thresholds, dismissedWarnings, thresholdsAvailable } = healthSettings;-- const memUsagePercent = Math.round((memStats.used / memStats.total) * 100);- const cpuLoad = os.loadavg()[0]; // 1-minute load average- const cpuCount = os.cpus().length;- const cpuUsagePercent = Math.round((cpuLoad / cpuCount) * 100);-- // Disk usage (root filesystem).- // bavail = blocks available to unprivileged users (what the user can actually fill).- // Derive used/usagePercent from the same figure so `used + free === total` and- // the UI's percent corresponds to the displayed `free`.- const parsedDisk = diskStats === failedProbe ? null : parseFilesystemStats(diskStats);- const disk = parsedDisk && {- total: parsedDisk.total,- used: parsedDisk.used,- free: parsedDisk.free,- usagePercent: parsedDisk.usagePercent,- };-- const dataDisk = parseFilesystemStats(dataDiskStats);- const sameFilesystem = filesystemDevices.every(device => device != null)- ? filesystemDevices[0] === filesystemDevices[1] : null;+ ]).then(([pm2Processes, appStatusSummary, cosStatus, cosPendingTaskIds, cosAgents, self, dbHealth, version, diskStats, dataDiskStats, filesystemDevices, memStats, healthSettings, forgeHealth, mediaCapacity, reviewerConfigHealth]) => ({+ pm2Processes, appStatusSummary, cosStatus, cosPendingTaskIds, cosAgents, self, dbHealth, version, diskStats, dataDiskStats, filesystemDevices, memStats, healthSettings, forgeHealth, mediaCapacity, reviewerConfigHealth+ }));+}- // Process status summary from PM2. Processes whose exit is expected (a desktop- // app the user closed) are excluded from the FAILURE-bearing counts: a quit game- // window would otherwise force overallHealth to 'critical' below and light up- // the dashboard widget and the OpenWorld HUD until the PM2 entry is manually- // cleared. `expectedExit` describes a process's exit *semantics*, not its- // liveness — so `online` counts every process, exempt or not, or a *running*- // desktop app would sit in `total` and in no status bucket at all and the- // dashboard would read "5/6 · all running" with all six up. Resource totals- // likewise cover every process. See issue #2991.- // When PM2 read fails, pm2Processes is null and we set processStats to null- // to signal unavailable rather than zero.- let processStats = null;- let annotated = [];- let supervised = [];- if (pm2Processes !== null) {- annotated = await annotateExpectedExit(pm2Processes);- supervised = annotated.filter(p => !p.expectedExit);- processStats = {+// Process status summary from PM2. Processes whose exit is expected (a desktop+// app the user closed) are excluded from the FAILURE-bearing counts: a quit game+// window would otherwise force overallHealth to 'critical' and light up the+// dashboard widget and the OpenWorld HUD until the PM2 entry is manually+// cleared. `expectedExit` describes a process's exit *semantics*, not its+// liveness — so `online` counts every process, exempt or not, or a *running*+// desktop app would sit in `total` and in no status bucket at all. Resource+// totals likewise cover every process. See issue #2991.+// A failed PM2 read yields processStats null (unavailable), not zeros.+async function deriveProcessStats(pm2Processes) {+ if (pm2Processes === null) return { processStats: null, supervised: [] };+ const annotated = await annotateExpectedExit(pm2Processes);+ const supervised = annotated.filter(p => !p.expectedExit);+ const sum = (list, key) => list.reduce((total, p) => total + (p[key] || 0), 0);+ return {+ supervised,+ processStats: {total: pm2Processes.length,online: annotated.filter(p => p.status === 'online').length,stopped: supervised.filter(p => p.status === 'stopped').length,@@ -179,206 +159,206 @@ export async function getSystemHealthSnapshot() {desktopExited: annotated.filter(p => p.expectedExit && ['errored', 'stopped'].includes(p.status)).length,- totalMemory: pm2Processes.reduce((sum, p) => sum + (p.memory || 0), 0),- totalCpu: pm2Processes.reduce((sum, p) => sum + (p.cpu || 0), 0),- totalRestarts: pm2Processes.reduce((sum, p) => sum + (p.restarts || 0), 0),- unstableRestarts: supervised.reduce((sum, p) => sum + (p.unstableRestarts || 0), 0)- };- }-- // App status summary — PM2-managed apps only (Xcode/iOS-native projects- // have no detectable runtime state, so they're tracked under `unmanaged`- // and excluded from the running denominator)- // Numeric placeholders preserve older clients' shape, not measurements.- const appStats = appStatusSummary === failedProbe- ? { total: 0, online: 0, stopped: 0, notStarted: 0, unknown: 0, unmanaged: 0, degraded: true, status: 'unavailable' }- : appStatusSummary;-- // Determine overall health status. Each condition below records its- // severity on the warning itself rather than mutating `overallHealth`- // inline, because a dismissed warning (see loadDismissedWarnings above)- // must not count toward the badge — overallHealth is derived once, after- // dismissals are filtered out, from whatever warnings remain visible.- const rawWarnings = [];-- // Memory occupancy and CPU load describe work, not a health failure.-- if (!thresholdsAvailable) {- rawWarnings.push({- type: 'health-settings',- severity: 'warning',- message: 'System health settings are unavailable; default thresholds are being used and saved warning dismissals were ignored.',- dismissible: false- });- }+ totalMemory: sum(pm2Processes, 'memory'),+ totalCpu: sum(pm2Processes, 'cpu'),+ totalRestarts: sum(pm2Processes, 'restarts'),+ unstableRestarts: sum(supervised, 'unstableRestarts')+ }+ };+}- if (diskStats === failedProbe) {- rawWarnings.push({ type: 'probe-unavailable', source: 'disk', status: 'unavailable', severity: 'warning', message: 'Disk status unavailable', dismissible: false });+function diskThresholdWarning(type, labelPrefix, usagePercent, thresholds) {+ if (usagePercent >= thresholds.diskCritical) {+ return { type, severity: 'critical', message: `${labelPrefix} usage at or above ${thresholds.diskCritical}%` };}- if (!dataDisk) {- rawWarnings.push({ type: 'probe-unavailable', source: 'data-disk', status: 'unavailable', severity: 'warning', message: 'Runtime data disk status unavailable', dismissible: false });- }- if (cosStatus === failedProbe) {- rawWarnings.push({ type: 'probe-unavailable', source: 'cos', status: 'unavailable', severity: 'warning', message: 'Chief of Staff status unavailable', dismissible: false });+ if (usagePercent >= thresholds.diskWarn) {+ return { type, severity: 'warning', message: `${labelPrefix} usage at or above ${thresholds.diskWarn}%` };}+ return null;+}++function cosProbeWarnings({ cosStatus, cosPendingTaskIds, cosAgents }) {+ if (cosStatus === FAILED_PROBE) return [probeFailureWarning('cos', 'Chief of Staff status unavailable')];// Queue and agent reads fail independently of getStatus(); only meaningful// while the daemon status itself was readable (otherwise 'cos' already warns).- if (cosStatus !== failedProbe) {- if (cosPendingTaskIds === failedProbe) {- rawWarnings.push({ type: 'probe-unavailable', source: 'cos-queue', status: 'unavailable', severity: 'warning', message: 'Chief of Staff queue unavailable', dismissible: false });- }- if (cosAgents === failedProbe) {- rawWarnings.push({ type: 'probe-unavailable', source: 'cos-agents', status: 'unavailable', severity: 'warning', message: 'Chief of Staff agent census unavailable', dismissible: false });- }- }- if (pm2Processes === null) {- rawWarnings.push({ type: 'probe-unavailable', source: 'pm2', status: 'unavailable', severity: 'warning', message: 'Process manager (PM2) status unavailable', dismissible: false });- }+ const warnings = [];+ if (cosPendingTaskIds === FAILED_PROBE) warnings.push(probeFailureWarning('cos-queue', 'Chief of Staff queue unavailable'));+ if (cosAgents === FAILED_PROBE) warnings.push(probeFailureWarning('cos-agents', 'Chief of Staff agent census unavailable'));+ return warnings;+}+function diskWarnings({ disk, dataDisk, sameFilesystem, thresholds }) {// Concurrent probes may sample a shared volume on either side of a threshold.// Keep one legacy warning identity and use the more severe available reading.const warningDisk = sameFilesystem === true && disk && dataDisk? (disk.usagePercent >= dataDisk.usagePercent ? disk : dataDisk) : disk;- if (warningDisk) {- if (warningDisk.usagePercent >= thresholds.diskCritical) {- rawWarnings.push({ type: 'disk', severity: 'critical', message: `Disk usage at or above ${thresholds.diskCritical}%` });- } else if (warningDisk.usagePercent >= thresholds.diskWarn) {- rawWarnings.push({ type: 'disk', severity: 'warning', message: `Disk usage at or above ${thresholds.diskWarn}%` });- }- }-+ const warnings = [];+ if (warningDisk) warnings.push(diskThresholdWarning('disk', 'Disk', warningDisk.usagePercent, thresholds));// Preserve the root warning identity and its saved dismissals on a shared volume.if (dataDisk && !(sameFilesystem === true && disk)) {- if (dataDisk.usagePercent >= thresholds.diskCritical) {- rawWarnings.push({ type: 'data-disk', severity: 'critical', message: `Runtime data disk usage at or above ${thresholds.diskCritical}%` });- } else if (dataDisk.usagePercent >= thresholds.diskWarn) {- rawWarnings.push({ type: 'data-disk', severity: 'warning', message: `Runtime data disk usage at or above ${thresholds.diskWarn}%` });- }+ warnings.push(diskThresholdWarning('data-disk', 'Runtime data disk', dataDisk.usagePercent, thresholds));}+ return warnings.filter(Boolean);+}- if (processStats) {- if (processStats.errored > 0) {- rawWarnings.push({ type: 'process', severity: 'critical', message: `${processStats.errored} process(es) errored` });- }-- if (processStats.unstableRestarts > 0) {- const crashing = supervised.filter(p => (p.unstableRestarts || 0) > 0).map(p => p.name);- const plural = processStats.unstableRestarts === 1 ? '' : 's';- rawWarnings.push({- type: 'restarts',- severity: 'warning',- message: `${processStats.unstableRestarts} crash-loop restart${plural} (${crashing.join(', ')})`- });- }+function processWarnings(processStats, supervised) {+ if (!processStats) return [];+ const warnings = [];+ if (processStats.errored > 0) {+ warnings.push({ type: 'process', severity: 'critical', message: `${processStats.errored} process(es) errored` });}-- // A degraded app summary means PM2 couldn't be read for one or more homes, so- // those apps' online/stopped status is unknown — surface it rather than letting- // the counts silently read as "everything not started."- if (appStatusSummary === failedProbe) {- rawWarnings.push({ type: 'probe-unavailable', source: 'apps', status: 'unavailable', severity: 'warning', message: 'Apps unavailable', dismissible: false });- } else if (appStats.degraded) {- const unknown = appStats.unknown || 0;- rawWarnings.push({ type: 'apps', severity: 'warning', message: `App status unavailable for ${unknown} app(s) — PM2 read failed` });+ if (processStats.unstableRestarts > 0) {+ const crashing = supervised.filter(p => (p.unstableRestarts || 0) > 0).map(p => p.name);+ const plural = processStats.unstableRestarts === 1 ? '' : 's';+ warnings.push({+ type: 'restarts',+ severity: 'warning',+ message: `${processStats.unstableRestarts} crash-loop restart${plural} (${crashing.join(', ')})`+ });}+ return warnings;+}+function databaseWarning(dbHealth) {if (!dbHealth.connected) {- rawWarnings.push({ type: 'database', severity: 'warning', message: `PostgreSQL disconnected${dbHealth.error ? `: ${dbHealth.error}` : ''}` });- } else if (!dbHealth.hasSchema) {- rawWarnings.push({ type: 'database', severity: 'warning', message: 'PostgreSQL connected but schema missing' });- } else if (dbHealth.hasCatalogSchema === false) {- rawWarnings.push({ type: 'database', severity: 'warning', message: 'PostgreSQL connected and schema available but Catalog tables missing — open Settings → Database to troubleshoot' });+ return { type: 'database', severity: 'warning', message: `PostgreSQL disconnected${dbHealth.error ? `: ${dbHealth.error}` : ''}` };}-- // A `gh` that cannot reach the forge does not fail loudly anywhere else: the- // call sites that read pull requests and issues swallow the error into an- // empty list, so a blocked or unauthenticated CLI looks exactly like a repo- // with nothing open — and an agent asked to file a PR reports success having- // filed none. Warn only when gh is present but unusable; an install that- // never had gh has opted out of those features rather than broken them.- if (!forgeHealth.ok && forgeHealth.status !== 'not-installed') {- rawWarnings.push({- type: 'forge',- severity: 'warning',- message: `GitHub CLI unusable (${forgeHealth.status})${forgeHealth.remedy ? ` — ${forgeHealth.remedy}` : ''}`- });+ if (!dbHealth.hasSchema) {+ return { type: 'database', severity: 'warning', message: 'PostgreSQL connected but schema missing' };}-- const reviewerConfigFaults = reviewerConfigHealth.configFaults || {};- if (Object.keys(reviewerConfigFaults).length) {- rawWarnings.push({- type: 'code-review',- severity: 'warning',- message: `Code review configuration needs attention for ${Object.keys(reviewerConfigFaults).join(', ')} — open Settings → Code Reviewers`- });+ if (dbHealth.hasCatalogSchema === false) {+ return { type: 'database', severity: 'warning', message: 'PostgreSQL connected and schema available but Catalog tables missing — open Settings → Database to troubleshoot' };}+ return null;+}++// A degraded app summary means PM2 couldn't be read for one or more homes, so+// those apps' online/stopped status is unknown — surface it rather than letting+// the counts silently read as "everything not started."+function appsWarning(appStatusSummary, appStats) {+ if (appStatusSummary === FAILED_PROBE) return probeFailureWarning('apps', 'Apps unavailable');+ if (!appStats.degraded) return null;+ return { type: 'apps', severity: 'warning', message: `App status unavailable for ${appStats.unknown || 0} app(s) — PM2 read failed` };+}++// A `gh` that cannot reach the forge does not fail loudly anywhere else: the+// call sites that read pull requests and issues swallow the error into an+// empty list, so a blocked or unauthenticated CLI looks exactly like a repo+// with nothing open — and an agent asked to file a PR reports success having+// filed none. Warn only when gh is present but unusable; an install that+// never had gh has opted out of those features rather than broken them.+function forgeWarning(forgeHealth) {+ if (forgeHealth.ok || forgeHealth.status === 'not-installed') return null;+ return {+ type: 'forge',+ severity: 'warning',+ message: `GitHub CLI unusable (${forgeHealth.status})${forgeHealth.remedy ? ` — ${forgeHealth.remedy}` : ''}`+ };+}++function reviewerConfigWarning(reviewerConfigHealth) {+ const faulty = Object.keys(reviewerConfigHealth.configFaults || {});+ if (!faulty.length) return null;+ return {+ type: 'code-review',+ severity: 'warning',+ message: `Code review configuration needs attention for ${faulty.join(', ')} — open Settings → Code Reviewers`+ };+}- if (!thresholdsAvailable) rawWarnings.forEach((warning) => { warning.dismissible = false; });+// Pure rule list: every condition records its severity on the warning itself+// rather than mutating overallHealth, because a dismissed warning must not+// count toward the badge — overallHealth derives from what remains visible.+// Memory occupancy and CPU load describe work, not a health failure.+function buildHealthWarnings(inputs) {+ const { thresholdsAvailable, diskStats, dataDisk, pm2Processes, processStats, supervised, appStatusSummary, appStats, dbHealth, forgeHealth, reviewerConfigHealth } = inputs;+ const warnings = [+ !thresholdsAvailable && {+ type: 'health-settings',+ severity: 'warning',+ message: 'System health settings are unavailable; default thresholds are being used and saved warning dismissals were ignored.',+ dismissible: false+ },+ diskStats === FAILED_PROBE && probeFailureWarning('disk', 'Disk status unavailable'),+ !dataDisk && probeFailureWarning('data-disk', 'Runtime data disk status unavailable'),+ ...cosProbeWarnings(inputs),+ pm2Processes === null && probeFailureWarning('pm2', 'Process manager (PM2) status unavailable'),+ ...diskWarnings(inputs),+ ...processWarnings(processStats, supervised),+ appsWarning(appStatusSummary, appStats),+ databaseWarning(dbHealth),+ forgeWarning(forgeHealth),+ reviewerConfigWarning(reviewerConfigHealth)+ ].filter(Boolean);+ if (!thresholdsAvailable) warnings.forEach((warning) => { warning.dismissible = false; });+ return warnings;+}- // A dismissal only stays applied while the warning it was recorded against- // is still current (same type AND same message) — see loadHealthSettings.- // Anything else (the condition cleared, or recurred with a different- // message) drops out of `dismissedWarnings` here so a genuinely new- // occurrence is never silently hidden by a stale record.- const nextDismissedWarnings = {};+// A dismissal only stays applied while the warning it was recorded against+// is still current (same type AND same message) — see loadHealthSettings.+// Anything else (the condition cleared, or recurred with a different message)+// is returned in `staleDismissals` so a genuinely new occurrence is never+// silently hidden by a stale record. `pendingPrunes` defaults to the module map.+function reconcileDismissals(rawWarnings, dismissedWarnings, thresholdsAvailable, pendingPrunes = pendingDismissalPrunes) {+ const kept = new Set();const warnings = [];for (const warning of rawWarnings) {const dismissal = dismissedWarnings[warning.type];- const pendingPrune = pendingDismissalPrunes.get(warning.type);- if (warning.dismissible !== false && dismissal?.message === warning.message && !sameDismissal(pendingPrune, dismissal)) {- nextDismissedWarnings[warning.type] = dismissal;+ if (warning.dismissible !== false && dismissal?.message === warning.message+ && !sameDismissal(pendingPrunes.get(warning.type), dismissal)) {+ kept.add(warning.type);continue;}warnings.push(warning);}+ const staleDismissals = thresholdsAvailable+ ? Object.entries(dismissedWarnings).filter(([type]) => !kept.has(type))+ : [];+ return { warnings, staleDismissals };+}- if (thresholdsAvailable) {- for (const [type, pending] of pendingDismissalPrunes) {- if (!Object.hasOwn(dismissedWarnings, type) || !sameDismissal(dismissedWarnings[type], pending)) {+// Owns the pendingDismissalPrunes bookkeeping and the settings write; a failed+// write keeps the entry pending so a matching recurrence stays visible.+async function pruneStaleDismissals(staleDismissals, dismissedWarnings) {+ for (const [type, pending] of pendingDismissalPrunes) {+ if (!Object.hasOwn(dismissedWarnings, type) || !sameDismissal(dismissedWarnings[type], pending)) {+ pendingDismissalPrunes.delete(type);+ }+ }+ if (!staleDismissals.length) return;+ for (const [type, dismissal] of staleDismissals) pendingDismissalPrunes.set(type, dismissal);+ try {+ const saved = await updateSettingsWith((current) => {+ const next = { ...(current.health?.dismissedWarnings || {}) };+ for (const [type, dismissal] of staleDismissals) {+ if (sameDismissal(next[type], dismissal)) delete next[type];+ }+ return patchHealth(current, { dismissedWarnings: next });+ });+ for (const [type, dismissal] of staleDismissals) {+ if (sameDismissal(pendingDismissalPrunes.get(type), dismissal)+ && !sameDismissal(saved?.health?.dismissedWarnings?.[type], dismissal)) {pendingDismissalPrunes.delete(type);}}-- const staleDismissals = Object.entries(dismissedWarnings)- .filter(([type]) => !Object.hasOwn(nextDismissedWarnings, type));- if (staleDismissals.length) {- for (const [type, dismissal] of staleDismissals) pendingDismissalPrunes.set(type, dismissal);- try {- const saved = await updateSettingsWith((current) => {- const next = { ...(current.health?.dismissedWarnings || {}) };- for (const [type, dismissal] of staleDismissals) {- if (sameDismissal(next[type], dismissal)) delete next[type];- }- return patchHealth(current, { dismissedWarnings: next });- });- for (const [type, dismissal] of staleDismissals) {- if (sameDismissal(pendingDismissalPrunes.get(type), dismissal)- && !sameDismissal(saved?.health?.dismissedWarnings?.[type], dismissal)) {- pendingDismissalPrunes.delete(type);- }- }- } catch (error) {- for (const [type] of staleDismissals) {- console.error(`❌ Failed to prune stale system health warning dismissal (type=${type}, error=${error?.code || 'write-failed'})`);- }- }+ } catch (error) {+ for (const [type] of staleDismissals) {+ console.error(`❌ Failed to prune stale system health warning dismissal (type=${type}, error=${error?.code || 'write-failed'})`);}}+}- const overallHealth = warnings.some(w => w.severity === 'critical')- ? 'critical'- : warnings.length > 0 ? 'warning' : 'healthy';-- // CoS status. Active and queued BOTH come off the one agent read when it is- // readable — mixing `getStatus()`'s tally with a separately-read queue lets the- // two skew, which is the defect one layer down (services/activeProcessing.js).- // A task a live agent already holds is active, not queued (lib/cosSpawnWindow.js).- // An unreadable list degrades to null — unknown, which the widget hides —- // rather than to a manufactured zero.- const agentsReadable = cosAgents !== failedProbe;- const queueReadable = cosPendingTaskIds !== failedProbe;- const heldByRunningAgent = agentsReadable ? runningAgentsByTaskId(cosAgents) : null;- const cosInfo = cosStatus && cosStatus !== failedProbe ? {+// CoS status. Active and queued BOTH come off the one agent read when it is+// readable — mixing `getStatus()`'s tally with a separately-read queue lets the+// two skew, which is the defect one layer down (services/activeProcessing.js).+// A task a live agent already holds is active, not queued (lib/cosSpawnWindow.js).+// An unreadable list degrades to null — unknown, which the widget hides —+// rather than to a manufactured zero.+function buildCosInfo({ cosStatus, cosPendingTaskIds, cosAgents }) {+ if (!cosStatus || cosStatus === FAILED_PROBE) return null;+ const agentsReadable = cosAgents !== FAILED_PROBE;+ const queueReadable = cosPendingTaskIds !== FAILED_PROBE;+ return {running: cosStatus.running,paused: cosStatus.paused,activeAgents: agentsReadable@@ -386,13 +366,93 @@ export async function getSystemHealthSnapshot() {: (cosStatus.activeAgents || 0),// Without the agent census the spawn-window subtraction is impossible, so// the count is unknown rather than an over-reported "reconciled" figure.- queuedTasks: queueReadable && agentsReadable ? unclaimedTaskIds(cosPendingTaskIds, heldByRunningAgent).length : null- } : null;+ queuedTasks: queueReadable && agentsReadable+ ? unclaimedTaskIds(cosPendingTaskIds, runningAgentsByTaskId(cosAgents)).length+ : null+ };+}+const withFormattedBytes = (fs) => fs && {+ total: fs.total,+ used: fs.used,+ free: fs.free,+ usagePercent: fs.usagePercent,+ totalFormatted: formatBytes(fs.total),+ usedFormatted: formatBytes(fs.used),+ freeFormatted: formatBytes(fs.free)+};++function topProcesses(pm2Processes) {+ if (pm2Processes === null) return null;+ return [...pm2Processes]+ .sort((a, b) => (b.memory || 0) - (a.memory || 0))+ .slice(0, 10)+ .map(p => ({+ name: p.name,+ status: p.status,+ memory: p.memory || 0,+ memoryFormatted: formatBytes(p.memory || 0),+ cpu: p.cpu || 0,+ restarts: p.restarts || 0,+ unstableRestarts: p.unstableRestarts || 0+ }));+}++function buildSystemSection({ memStats, disk, dataDisk, sameFilesystem }) {+ const cpuLoad = os.loadavg()[0]; // 1-minute load average+ const cpuCount = os.cpus().length;const uptime = process.uptime();- const uptimeFormatted = formatDuration(uptime * 1000);+ return {+ uptime,+ uptimeFormatted: formatDuration(uptime * 1000),+ memory: {+ total: memStats.total,+ used: memStats.used,+ free: memStats.free,+ usagePercent: Math.round((memStats.used / memStats.total) * 100),+ totalFormatted: formatBytes(memStats.total),+ usedFormatted: formatBytes(memStats.used),+ freeFormatted: formatBytes(memStats.free)+ },+ cpu: {+ cores: cpuCount,+ loadAvg1m: cpuLoad,+ usagePercent: Math.round((cpuLoad / cpuCount) * 100)+ },+ disk: withFormattedBytes(disk),+ dataDisk: withFormattedBytes(dataDisk),+ sameFilesystem+ };+}- const _responseTime = Date.now() - startTime;+export async function getSystemHealthSnapshot() {+ const probes = await probeHealthInputs();+ const { pm2Processes, appStatusSummary, self, dbHealth, version, diskStats, dataDiskStats, filesystemDevices, memStats, healthSettings, forgeHealth, mediaCapacity, reviewerConfigHealth } = probes;+ const { thresholds, dismissedWarnings, thresholdsAvailable } = healthSettings;++ // bavail = blocks available to unprivileged users (what the user can actually fill).+ // Derive used/usagePercent from the same figure so `used + free === total` and+ // the UI's percent corresponds to the displayed `free`.+ const disk = diskStats === FAILED_PROBE ? null : parseFilesystemStats(diskStats);+ const dataDisk = parseFilesystemStats(dataDiskStats);+ const sameFilesystem = filesystemDevices.every(device => device != null)+ ? filesystemDevices[0] === filesystemDevices[1] : null;++ const { processStats, supervised } = await deriveProcessStats(pm2Processes);+ // PM2-managed apps only (Xcode/iOS-native projects have no detectable runtime+ // state, so they're `unmanaged`). Numeric placeholders preserve older+ // clients' shape, not measurements.+ const appStats = appStatusSummary === FAILED_PROBE+ ? { total: 0, online: 0, stopped: 0, notStarted: 0, unknown: 0, unmanaged: 0, degraded: true, status: 'unavailable' }+ : appStatusSummary;++ const rawWarnings = buildHealthWarnings({ ...probes, ...healthSettings, disk, dataDisk, sameFilesystem, processStats, supervised, appStats });+ const { warnings, staleDismissals } = reconcileDismissals(rawWarnings, dismissedWarnings, thresholdsAvailable);+ if (thresholdsAvailable) await pruneStaleDismissals(staleDismissals, dismissedWarnings);++ const overallHealth = warnings.some(w => w.severity === 'critical')+ ? 'critical'+ : warnings.length > 0 ? 'warning' : 'healthy';return {timestamp: new Date().toISOString(),@@ -408,63 +468,23 @@ export async function getSystemHealthSnapshot() {accepted: false,},warnings,- system: {- uptime,- uptimeFormatted,- memory: {- total: memStats.total,- used: memStats.used,- free: memStats.free,- usagePercent: memUsagePercent,- totalFormatted: formatBytes(memStats.total),- usedFormatted: formatBytes(memStats.used),- freeFormatted: formatBytes(memStats.free)- },- cpu: {- cores: cpuCount,- loadAvg1m: cpuLoad,- usagePercent: cpuUsagePercent- },- disk: disk ? {- total: disk.total,- used: disk.used,- free: disk.free,- usagePercent: disk.usagePercent,- totalFormatted: formatBytes(disk.total),- usedFormatted: formatBytes(disk.used),- freeFormatted: formatBytes(disk.free)- } : null,- dataDisk: dataDisk ? {- total: dataDisk.total,- used: dataDisk.used,- free: dataDisk.free,- usagePercent: dataDisk.usagePercent,- totalFormatted: formatBytes(dataDisk.total),- usedFormatted: formatBytes(dataDisk.used),- freeFormatted: formatBytes(dataDisk.free)- } : null,- sameFilesystem- },+ system: buildSystemSection({ memStats, disk, dataDisk, sameFilesystem }),processes: processStats,apps: appStats,- cos: cosInfo,+ cos: buildCosInfo(probes),media: mediaCapacity,database: dbHealth,forge: forgeHealth,codeReview: reviewerConfigHealth,thresholds: thresholdsAvailable ? thresholds : undefined,thresholdsAvailable,- topProcesses: pm2Processes === null ? null : [...pm2Processes]- .sort((a, b) => (b.memory || 0) - (a.memory || 0))- .slice(0, 10)- .map(p => ({- name: p.name,- status: p.status,- memory: p.memory || 0,- memoryFormatted: formatBytes(p.memory || 0),- cpu: p.cpu || 0,- restarts: p.restarts || 0,- unstableRestarts: p.unstableRestarts || 0- }))+ topProcesses: topProcesses(pm2Processes)};}++// Test hooks for the pure rule and dismissal steps.+export {+ buildHealthWarnings as _buildHealthWarnings,+ reconcileDismissals as _reconcileDismissals,+ pruneStaleDismissals as _pruneStaleDismissals+};diff --git a/server/services/systemHealthSnapshot.test.js b/server/services/systemHealthSnapshot.test.jsnew file mode 100644index 0000000000..46042d74a1--- /dev/null+++ b/server/services/systemHealthSnapshot.test.js@@ -0,0 +1,112 @@+import { describe, it, expect, vi } from 'vitest';++// The pure rule/dismissal functions need none of the probes; stub the modules+// so importing the file does not instantiate the CoS/DB/PM2 graph.+vi.mock('./pm2.js', () => ({ listProcessesStrict: vi.fn() }));+vi.mock('./appProcessStatus.js', () => ({ getAppStatusSummary: vi.fn(), annotateExpectedExit: vi.fn() }));+vi.mock('./cos.js', () => ({}));+vi.mock('./instanceIdentity.js', () => ({ getSelf: vi.fn() }));+vi.mock('../lib/db.js', () => ({ checkHealth: vi.fn() }));+vi.mock('./updateChecker.js', () => ({ getCurrentVersion: vi.fn() }));+vi.mock('../lib/memoryStats.js', () => ({ getMemoryStats: vi.fn() }));+vi.mock('./github.js', () => ({ checkGhHealth: vi.fn() }));+vi.mock('./mediaCapacity.js', () => ({ getMediaCapacity: vi.fn() }));+const updateSettingsWith = vi.hoisted(() => vi.fn());+vi.mock('./settings.js', () => ({ getSettingsWithStatus: vi.fn(), updateSettingsWith }));++const {+ _buildHealthWarnings: buildHealthWarnings,+ _reconcileDismissals: reconcileDismissals,+ _pruneStaleDismissals: pruneStaleDismissals+} = await import('./systemHealthSnapshot.js');++const healthyInputs = (overrides = {}) => ({+ thresholdsAvailable: true,+ thresholds: { diskWarn: 90, diskCritical: 98 },+ diskStats: {},+ disk: { usagePercent: 10 },+ dataDisk: { usagePercent: 10 },+ sameFilesystem: false,+ pm2Processes: [],+ processStats: { errored: 0, unstableRestarts: 0 },+ supervised: [],+ appStatusSummary: {},+ appStats: { degraded: false },+ cosStatus: {},+ cosPendingTaskIds: [],+ cosAgents: [],+ dbHealth: { connected: true, hasSchema: true },+ forgeHealth: { ok: true },+ reviewerConfigHealth: { configFaults: {} },+ ...overrides+});++describe('buildHealthWarnings', () => {+ it('emits nothing for a healthy snapshot', () => {+ expect(buildHealthWarnings(healthyInputs())).toEqual([]);+ });++ it('reports unavailable probes as non-dismissible', () => {+ const sample = buildHealthWarnings(healthyInputs({ pm2Processes: null, processStats: null, dataDisk: null }));+ expect(sample.map(w => w.source)).toEqual(['data-disk', 'pm2']);+ expect(sample.every(w => w.dismissible === false && w.type === 'probe-unavailable')).toBe(true);+ });++ it('uses one threshold ladder for root and data disks', () => {+ const warnings = buildHealthWarnings(healthyInputs({+ disk: { usagePercent: 91 }, dataDisk: { usagePercent: 99 }+ }));+ expect(warnings).toEqual([+ { type: 'disk', severity: 'warning', message: 'Disk usage at or above 90%' },+ { type: 'data-disk', severity: 'critical', message: 'Runtime data disk usage at or above 98%' }+ ]);+ });++ it('keeps one warning identity on a shared volume and forces non-dismissible when settings are unavailable', () => {+ const warnings = buildHealthWarnings(healthyInputs({+ thresholdsAvailable: false, sameFilesystem: true,+ disk: { usagePercent: 50 }, dataDisk: { usagePercent: 95 }+ }));+ expect(warnings.map(w => w.type)).toEqual(['health-settings', 'disk']);+ expect(warnings.every(w => w.dismissible === false)).toBe(true);+ });+});++describe('reconcileDismissals', () => {+ const warning = { type: 'disk', severity: 'warning', message: 'Disk usage at or above 90%' };++ it('suppresses a dismissal that is still current', () => {+ const dismissed = { disk: { message: warning.message, dismissedAt: 't' } };+ expect(reconcileDismissals([warning], dismissed, true, new Map()))+ .toEqual({ warnings: [], staleDismissals: [] });+ });++ it('shows a changed message again and marks the old dismissal stale', () => {+ const dismissed = { disk: { message: 'Disk usage at or above 80%', dismissedAt: 't' } };+ const result = reconcileDismissals([warning], dismissed, true, new Map());+ expect(result.warnings).toEqual([warning]);+ expect(result.staleDismissals).toEqual([['disk', dismissed.disk]]);+ });++ it('keeps a warning visible while its dismissal awaits a prune', () => {+ const dismissal = { message: warning.message, dismissedAt: 't' };+ const result = reconcileDismissals([warning], { disk: dismissal }, true, new Map([['disk', { ...dismissal }]]));+ expect(result.warnings).toEqual([warning]);+ expect(result.staleDismissals).toEqual([['disk', dismissal]]);+ });++ it('never reports stale dismissals while settings are unavailable', () => {+ expect(reconcileDismissals([], { disk: { message: 'x' } }, false, new Map()).staleDismissals).toEqual([]);+ });+});++describe('pruneStaleDismissals', () => {+ it('logs and swallows a failed settings write', async () => {+ updateSettingsWith.mockRejectedValueOnce(Object.assign(new Error('boom'), { code: 'EIO' }));+ const spy = vi.spyOn(console, 'error').mockImplementation(() => {});+ const dismissal = { message: 'm' };+ await expect(pruneStaleDismissals([['gone', dismissal]], { gone: dismissal })).resolves.toBeUndefined();+ expect(spy).toHaveBeenCalledWith(expect.stringContaining('type=gone, error=EIO'));+ spy.mockRestore();+ });+});