Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: serialize critical-health notification creation and recovery (#10724)
Adam Eivy6b04acc0aa0e
diff --git a/server/services/healthCriticalNotifier.js b/server/services/healthCriticalNotifier.jsindex 1405c05d9a..169e393fee 100644--- a/server/services/healthCriticalNotifier.js+++ b/server/services/healthCriticalNotifier.js@@ -10,34 +10,48 @@* card is deduped on the issue SET: an unchanged set raises nothing, a changed* set replaces the old card, and a `health:check` with no error-level issues* retracts it. Machine-local — `data/notifications.json` is never federated.+ *+ * The notification store locks each individual write, but a transition here is+ * several awaits (exists → remove → add), and cos.js launches the two entry+ * points from independent event listeners without awaiting either. One+ * module-owned queue runs each whole transition in the order it was received, so+ * two identical events cannot both pass the exists check and a stale failure+ * cannot land its card after a newer recovery.*/+import { createMutex } from '../lib/asyncMutex.js';import { addNotification, exists, removeByMetadata, NOTIFICATION_TYPES, PRIORITY_LEVELS } from './notifications.js';const SOURCE = 'cos-health-critical';+// A rejected transition still releases the queue (createMutex uses try/finally),+// so later work runs and the rejection stays visible to the caller's catch.+const inTransition = createMutex();+const issueKey = (issues) => [...new Set(issues.map(i => i.message))].sort().join('\n');/*** @param {Array<{ message: string, category?: string }>} issues - `health:critical` payload* @returns {Promise<boolean>} whether a card was raised*/-export async function notifyCriticalHealth(issues) {- if (!Array.isArray(issues) || issues.length === 0) return false;- const key = issueKey(issues);- if (await exists(NOTIFICATION_TYPES.HEALTH_ISSUE, 'healthCriticalKey', key)) return false;- await removeByMetadata('source', SOURCE);- const shown = issues.slice(0, 5).map(i => i.message);- const more = issues.length > shown.length ? ` (+${issues.length - shown.length} more)` : '';- await addNotification({- type: NOTIFICATION_TYPES.HEALTH_ISSUE,- title: `Critical health: ${issues.length} issue${issues.length === 1 ? '' : 's'}`,- description: `${shown.join('; ')}${more}`,- priority: PRIORITY_LEVELS.CRITICAL,- link: '/system-resources/overview',- metadata: { source: SOURCE, healthCriticalKey: key },+export function notifyCriticalHealth(issues) {+ if (!Array.isArray(issues) || issues.length === 0) return Promise.resolve(false);+ return inTransition(async () => {+ const key = issueKey(issues);+ if (await exists(NOTIFICATION_TYPES.HEALTH_ISSUE, 'healthCriticalKey', key)) return false;+ await removeByMetadata('source', SOURCE);+ const shown = issues.slice(0, 5).map(i => i.message);+ const more = issues.length > shown.length ? ` (+${issues.length - shown.length} more)` : '';+ await addNotification({+ type: NOTIFICATION_TYPES.HEALTH_ISSUE,+ title: `Critical health: ${issues.length} issue${issues.length === 1 ? '' : 's'}`,+ description: `${shown.join('; ')}${more}`,+ priority: PRIORITY_LEVELS.CRITICAL,+ link: '/system-resources/overview',+ metadata: { source: SOURCE, healthCriticalKey: key },+ });+ return true;});- return true;}/**@@ -46,8 +60,10 @@ export async function notifyCriticalHealth(issues) {* @param {{ issues?: Array<{ type?: string }> }} check - `health:check` payload* @returns {Promise<boolean>} whether a card was retracted*/-export async function clearCriticalHealthIfRecovered(check) {- if ((check?.issues ?? []).some(i => i.type === 'error')) return false;- const { removed } = await removeByMetadata('source', SOURCE);- return removed > 0;+export function clearCriticalHealthIfRecovered(check) {+ if ((check?.issues ?? []).some(i => i.type === 'error')) return Promise.resolve(false);+ return inTransition(async () => {+ const { removed } = await removeByMetadata('source', SOURCE);+ return removed > 0;+ });}diff --git a/server/services/healthCriticalNotifier.test.js b/server/services/healthCriticalNotifier.test.jsindex 0bbcd771d9..14974ccc60 100644--- a/server/services/healthCriticalNotifier.test.js+++ b/server/services/healthCriticalNotifier.test.js@@ -63,3 +63,98 @@ describe('clearCriticalHealthIfRecovered', () => {expect(removeByMetadata).not.toHaveBeenCalled();});});++// Ordering: cos.js launches both entry points from independent listeners without+// awaiting either. A synthetic store serializes each individual call (as the real+// notification store does) but lets a test hold any one call open.+describe('transition ordering', () => {+ const SOURCE = 'cos-health-critical';+ const other = [{ type: 'error', category: 'processes', message: 'Disk unreadable' }];+ let cards;+ let gate; // optional (op) => Promise to await before the op takes effect++ const deferred = () => {+ let release;+ const promise = new Promise(r => { release = r; });+ return { promise, release };+ };+ // Let queued microtasks run so a blocked transition reaches its held call.+ const flush = async () => { for (let i = 0; i < 10; i++) await Promise.resolve(); };++ beforeEach(() => {+ cards = [];+ gate = null;+ exists.mockImplementation(async (_type, _field, key) => {+ const hit = cards.some(c => c.metadata.healthCriticalKey === key);+ await gate?.('exists');+ return hit;+ });+ removeByMetadata.mockImplementation(async (_field, value) => {+ await gate?.('remove');+ const before = cards.length;+ cards = cards.filter(c => c.metadata.source !== value);+ return { success: true, removed: before - cards.length };+ });+ addNotification.mockImplementation(async (card) => {+ await gate?.('add');+ cards.push(card);+ return card;+ });+ });++ it('raises exactly one card for concurrent identical critical events', async () => {+ const hold = deferred();+ gate = (op) => (op === 'exists' ? hold.promise : undefined);+ const first = notifyCriticalHealth(issues);+ const second = notifyCriticalHealth([...issues].reverse());+ await flush();+ hold.release();+ expect(await Promise.all([first, second])).toEqual([true, false]);+ expect(cards).toHaveLength(1);+ });++ it('does not let a delayed failure land its card after a later recovery', async () => {+ const hold = deferred();+ let held = false;+ gate = (op) => {+ if (op === 'exists' && !held) { held = true; return hold.promise; }+ };+ const failure = notifyCriticalHealth(issues);+ const recovery = clearCriticalHealthIfRecovered({ issues: [] });+ await flush();+ hold.release();+ await Promise.all([failure, recovery]);+ expect(cards).toHaveLength(0);+ });++ it('leaves one card for the latest state across changed sets and a new failure after recovery', async () => {+ const hold = deferred();+ let held = false;+ gate = (op) => {+ if (op === 'add' && !held) { held = true; return hold.promise; }+ };+ const calls = [+ notifyCriticalHealth(issues),+ notifyCriticalHealth(other),+ clearCriticalHealthIfRecovered({ issues: [] }),+ notifyCriticalHealth(issues),+ ];+ await flush();+ hold.release();+ await Promise.all(calls);+ expect(cards).toHaveLength(1);+ expect(cards[0].metadata.healthCriticalKey).toBe(issueKeyOf(issues));+ });++ it('releases the queue after a rejected transition and surfaces the error to its caller', async () => {+ exists.mockRejectedValueOnce(new Error('store unavailable'));+ const failing = notifyCriticalHealth(issues);+ const next = notifyCriticalHealth(issues);+ await expect(failing).rejects.toThrow('store unavailable');+ expect(await next).toBe(true);+ expect(cards).toHaveLength(1);+ expect(cards[0].metadata.source).toBe(SOURCE);+ });+});++const issueKeyOf = (list) => [...new Set(list.map(i => i.message))].sort().join('\n');