Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: serialize critical-health notification creation and recovery (#10724)

Adam Eivy6b04acc0aa0e
diff --git a/server/services/healthCriticalNotifier.js b/server/services/healthCriticalNotifier.js
index 1405c05d9a..169e393fee 100644
--- a/server/services/healthCriticalNotifier.js
+++ b/server/services/healthCriticalNotifier.js
@@ -10,34 +10,48 @@
* card is deduped on the issue SET: an unchanged set raises nothing, a changed
* set replaces the old card, and a `health:check` with no error-level issues
* retracts it. Machine-local — `data/notifications.json` is never federated.
+ *
+ * The notification store locks each individual write, but a transition here is
+ * several awaits (exists → remove → add), and cos.js launches the two entry
+ * points from independent event listeners without awaiting either. One
+ * module-owned queue runs each whole transition in the order it was received, so
+ * two identical events cannot both pass the exists check and a stale failure
+ * cannot land its card after a newer recovery.
*/
+import { createMutex } from '../lib/asyncMutex.js';
import { addNotification, exists, removeByMetadata, NOTIFICATION_TYPES, PRIORITY_LEVELS } from './notifications.js';
const SOURCE = 'cos-health-critical';
+// A rejected transition still releases the queue (createMutex uses try/finally),
+// so later work runs and the rejection stays visible to the caller's catch.
+const inTransition = createMutex();
+
const issueKey = (issues) => [...new Set(issues.map(i => i.message))].sort().join('\n');
/**
* @param {Array<{ message: string, category?: string }>} issues - `health:critical` payload
* @returns {Promise<boolean>} whether a card was raised
*/
-export async function notifyCriticalHealth(issues) {
- if (!Array.isArray(issues) || issues.length === 0) return false;
- const key = issueKey(issues);
- if (await exists(NOTIFICATION_TYPES.HEALTH_ISSUE, 'healthCriticalKey', key)) return false;
- await removeByMetadata('source', SOURCE);
- const shown = issues.slice(0, 5).map(i => i.message);
- const more = issues.length > shown.length ? ` (+${issues.length - shown.length} more)` : '';
- await addNotification({
- type: NOTIFICATION_TYPES.HEALTH_ISSUE,
- title: `Critical health: ${issues.length} issue${issues.length === 1 ? '' : 's'}`,
- description: `${shown.join('; ')}${more}`,
- priority: PRIORITY_LEVELS.CRITICAL,
- link: '/system-resources/overview',
- metadata: { source: SOURCE, healthCriticalKey: key },
+export function notifyCriticalHealth(issues) {
+ if (!Array.isArray(issues) || issues.length === 0) return Promise.resolve(false);
+ return inTransition(async () => {
+ const key = issueKey(issues);
+ if (await exists(NOTIFICATION_TYPES.HEALTH_ISSUE, 'healthCriticalKey', key)) return false;
+ await removeByMetadata('source', SOURCE);
+ const shown = issues.slice(0, 5).map(i => i.message);
+ const more = issues.length > shown.length ? ` (+${issues.length - shown.length} more)` : '';
+ await addNotification({
+ type: NOTIFICATION_TYPES.HEALTH_ISSUE,
+ title: `Critical health: ${issues.length} issue${issues.length === 1 ? '' : 's'}`,
+ description: `${shown.join('; ')}${more}`,
+ priority: PRIORITY_LEVELS.CRITICAL,
+ link: '/system-resources/overview',
+ metadata: { source: SOURCE, healthCriticalKey: key },
+ });
+ return true;
});
- return true;
}
/**
@@ -46,8 +60,10 @@ export async function notifyCriticalHealth(issues) {
* @param {{ issues?: Array<{ type?: string }> }} check - `health:check` payload
* @returns {Promise<boolean>} whether a card was retracted
*/
-export async function clearCriticalHealthIfRecovered(check) {
- if ((check?.issues ?? []).some(i => i.type === 'error')) return false;
- const { removed } = await removeByMetadata('source', SOURCE);
- return removed > 0;
+export function clearCriticalHealthIfRecovered(check) {
+ if ((check?.issues ?? []).some(i => i.type === 'error')) return Promise.resolve(false);
+ return inTransition(async () => {
+ const { removed } = await removeByMetadata('source', SOURCE);
+ return removed > 0;
+ });
}
diff --git a/server/services/healthCriticalNotifier.test.js b/server/services/healthCriticalNotifier.test.js
index 0bbcd771d9..14974ccc60 100644
--- a/server/services/healthCriticalNotifier.test.js
+++ b/server/services/healthCriticalNotifier.test.js
@@ -63,3 +63,98 @@ describe('clearCriticalHealthIfRecovered', () => {
expect(removeByMetadata).not.toHaveBeenCalled();
});
});
+
+// Ordering: cos.js launches both entry points from independent listeners without
+// awaiting either. A synthetic store serializes each individual call (as the real
+// notification store does) but lets a test hold any one call open.
+describe('transition ordering', () => {
+ const SOURCE = 'cos-health-critical';
+ const other = [{ type: 'error', category: 'processes', message: 'Disk unreadable' }];
+ let cards;
+ let gate; // optional (op) => Promise to await before the op takes effect
+
+ const deferred = () => {
+ let release;
+ const promise = new Promise(r => { release = r; });
+ return { promise, release };
+ };
+ // Let queued microtasks run so a blocked transition reaches its held call.
+ const flush = async () => { for (let i = 0; i < 10; i++) await Promise.resolve(); };
+
+ beforeEach(() => {
+ cards = [];
+ gate = null;
+ exists.mockImplementation(async (_type, _field, key) => {
+ const hit = cards.some(c => c.metadata.healthCriticalKey === key);
+ await gate?.('exists');
+ return hit;
+ });
+ removeByMetadata.mockImplementation(async (_field, value) => {
+ await gate?.('remove');
+ const before = cards.length;
+ cards = cards.filter(c => c.metadata.source !== value);
+ return { success: true, removed: before - cards.length };
+ });
+ addNotification.mockImplementation(async (card) => {
+ await gate?.('add');
+ cards.push(card);
+ return card;
+ });
+ });
+
+ it('raises exactly one card for concurrent identical critical events', async () => {
+ const hold = deferred();
+ gate = (op) => (op === 'exists' ? hold.promise : undefined);
+ const first = notifyCriticalHealth(issues);
+ const second = notifyCriticalHealth([...issues].reverse());
+ await flush();
+ hold.release();
+ expect(await Promise.all([first, second])).toEqual([true, false]);
+ expect(cards).toHaveLength(1);
+ });
+
+ it('does not let a delayed failure land its card after a later recovery', async () => {
+ const hold = deferred();
+ let held = false;
+ gate = (op) => {
+ if (op === 'exists' && !held) { held = true; return hold.promise; }
+ };
+ const failure = notifyCriticalHealth(issues);
+ const recovery = clearCriticalHealthIfRecovered({ issues: [] });
+ await flush();
+ hold.release();
+ await Promise.all([failure, recovery]);
+ expect(cards).toHaveLength(0);
+ });
+
+ it('leaves one card for the latest state across changed sets and a new failure after recovery', async () => {
+ const hold = deferred();
+ let held = false;
+ gate = (op) => {
+ if (op === 'add' && !held) { held = true; return hold.promise; }
+ };
+ const calls = [
+ notifyCriticalHealth(issues),
+ notifyCriticalHealth(other),
+ clearCriticalHealthIfRecovered({ issues: [] }),
+ notifyCriticalHealth(issues),
+ ];
+ await flush();
+ hold.release();
+ await Promise.all(calls);
+ expect(cards).toHaveLength(1);
+ expect(cards[0].metadata.healthCriticalKey).toBe(issueKeyOf(issues));
+ });
+
+ it('releases the queue after a rejected transition and surfaces the error to its caller', async () => {
+ exists.mockRejectedValueOnce(new Error('store unavailable'));
+ const failing = notifyCriticalHealth(issues);
+ const next = notifyCriticalHealth(issues);
+ await expect(failing).rejects.toThrow('store unavailable');
+ expect(await next).toBe(true);
+ expect(cards).toHaveLength(1);
+ expect(cards[0].metadata.source).toBe(SOURCE);
+ });
+});
+
+const issueKeyOf = (list) => [...new Set(list.map(i => i.message))].sort().join('\n');