Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop concurrent Review Hub writes from silently dropping items (#10911)

Adam Eivydc93ce7b0ac5
diff --git a/server/services/review.concurrency.test.js b/server/services/review.concurrency.test.js
new file mode 100644
index 0000000000..e92382c8e2
--- /dev/null
+++ b/server/services/review.concurrency.test.js
@@ -0,0 +1,130 @@
+/**
+ * Review Hub write serialization (#10911).
+ *
+ * Runs the real review.js against a real temp data root (no fs mocks): the
+ * lost-update race only exists when two load -> mutate -> save turns actually
+ * overlap on the filesystem, which a mocked store cannot reproduce.
+ */
+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdir, rm, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';
+
+const { tempRoot, makeProxy, cleanup } = mockPathsDataRoot({ prefix: 'portos-review-concurrency-' });
+
+vi.mock('../lib/fileUtils.js', async () => {
+ const actual = await vi.importActual('../lib/fileUtils.js');
+ return makeProxy(actual);
+});
+
+const { createItem, getItems, dismissItem, reopenItem, bulkUpdateStatus, updateItem, deleteItem, reviewEvents } =
+ await import('./review.js');
+
+const reviewDir = join(tempRoot, 'review');
+const archiveFile = join(reviewDir, 'archive.json');
+
+const archivedTodo = {
+ id: 'archived-1',
+ type: 'todo',
+ title: 'Archived example',
+ description: '',
+ status: 'dismissed',
+ metadata: {},
+ createdAt: '2020-01-01T00:00:00.000Z',
+ updatedAt: '2020-01-02T00:00:00.000Z',
+};
+
+const todo = (title) => ({ type: 'todo', title });
+
+describe('review service write serialization', () => {
+ beforeEach(async () => {
+ await rm(reviewDir, { recursive: true, force: true });
+ });
+
+ afterEach(() => {
+ reviewEvents.removeAllListeners('item:created');
+ });
+
+ afterAll(cleanup);
+
+ it('persists every item from overlapping createItem calls and emits one event each', async () => {
+ const created = [];
+ reviewEvents.on('item:created', (item) => created.push(item.id));
+
+ const results = await Promise.all(
+ Array.from({ length: 8 }, (_, i) => createItem(todo(`Item ${i}`))),
+ );
+
+ const stored = await getItems();
+ expect(stored).toHaveLength(8);
+ expect(new Set(stored.map((i) => i.id))).toEqual(new Set(results.map((i) => i.id)));
+ expect(created).toHaveLength(8);
+ });
+
+ it('keeps both a new item and a dismissal of another item', async () => {
+ const existing = await createItem(todo('Existing'));
+
+ const [fresh] = await Promise.all([createItem(todo('Fresh')), dismissItem(existing.id)]);
+
+ const stored = await getItems();
+ expect(stored.find((i) => i.id === fresh.id)?.status).toBe('pending');
+ expect(stored.find((i) => i.id === existing.id)?.status).toBe('dismissed');
+ });
+
+ it('keeps an item created while bulkUpdateStatus runs', async () => {
+ const a = await createItem(todo('A'));
+ const b = await createItem(todo('B'));
+
+ const [fresh, updated] = await Promise.all([
+ createItem(todo('Fresh')),
+ bulkUpdateStatus({ ids: [a.id, b.id], status: 'dismissed' }),
+ ]);
+
+ expect(updated).toHaveLength(2);
+ const stored = await getItems();
+ expect(stored.find((i) => i.id === fresh.id)?.status).toBe('pending');
+ expect(stored.filter((i) => i.status === 'dismissed')).toHaveLength(2);
+ });
+
+ it('loses neither a reopened archived item nor a concurrent create', async () => {
+ await mkdir(reviewDir, { recursive: true });
+ await writeFile(archiveFile, JSON.stringify([archivedTodo]));
+
+ const [fresh, reopened] = await Promise.all([createItem(todo('Fresh')), reopenItem(archivedTodo.id)]);
+
+ const stored = await getItems();
+ expect(stored.find((i) => i.id === fresh.id)?.status).toBe('pending');
+ expect(stored.find((i) => i.id === reopened.id)?.status).toBe('pending');
+ // Gone from cold storage: the live copy is the only one.
+ expect(stored.filter((i) => i.id === archivedTodo.id)).toHaveLength(1);
+ });
+
+ it('serializes edits and deletes against creates on the same file', async () => {
+ const keep = await createItem(todo('Keep'));
+ const drop = await createItem(todo('Drop'));
+
+ const [fresh] = await Promise.all([
+ createItem(todo('Fresh')),
+ updateItem(keep.id, { title: 'Renamed' }),
+ deleteItem(drop.id),
+ ]);
+
+ const stored = await getItems();
+ expect(stored.map((i) => i.id).sort()).toEqual([keep.id, fresh.id].sort());
+ expect(stored.find((i) => i.id === keep.id)?.title).toBe('Renamed');
+ });
+
+ it('does not deadlock when an event listener calls a mutator synchronously', async () => {
+ let follow;
+ reviewEvents.once('item:created', () => {
+ follow = createItem(todo('Follow-up'));
+ });
+
+ const first = await createItem(todo('First'));
+ await follow;
+
+ const stored = await getItems();
+ expect(stored.map((i) => i.title).sort()).toEqual(['First', 'Follow-up']);
+ expect(stored.some((i) => i.id === first.id)).toBe(true);
+ });
+});
diff --git a/server/services/review.js b/server/services/review.js
index a015d95eac..e713ccd3c6 100644
--- a/server/services/review.js
+++ b/server/services/review.js
@@ -10,6 +10,7 @@ import { join } from 'path';
import { v4 as uuidv4 } from '../lib/uuid.js';
import { EventEmitter } from 'events';
import { ensureDir, PATHS, readJSONFile, atomicWrite } from '../lib/fileUtils.js';
+import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
import { cosEvents } from './cosEvents.js';
import { GOAL_FIDELITY_HOLD_EVENT, formatGoalFidelitySummary } from '../lib/goalFidelity.js';
@@ -57,6 +58,27 @@ const ARCHIVE_FILE = join(DATA_DIR, 'archive.json');
export const reviewEvents = new EventEmitter();
+// One write tail for BOTH items.json and archive.json: `reopenItem` and
+// `applyRetention` mutate the pair together, and `atomicWrite` only prevents a
+// torn file — it does not stop two overlapping load -> mutate -> save turns
+// from replacing each other's change (#10911). Every mutator runs its whole
+// read-modify-write region through `runMutation`.
+const queueWrite = createFileWriteQueue();
+
+/**
+ * Run a load -> mutate -> save region serialized against every other review
+ * mutation, then emit its events AFTER the queue turn has released. The region
+ * returns `{ result, events }` where `events` is a list of `[name, payload]`.
+ * Emitting outside the critical section keeps a `reviewEvents` listener that
+ * calls a mutator synchronously from re-entering (and deadlocking on) the
+ * queue it is still holding. Regions must never await another queued mutator.
+ */
+async function runMutation(region) {
+ const { result, events = [] } = await queueWrite(region);
+ for (const [name, payload] of events) reviewEvents.emit(name, payload);
+ return result;
+}
+
// Valid item types and statuses
const ITEM_TYPES = ['alert', 'todo', 'briefing', 'cos'];
const ITEM_STATUSES = ['pending', 'completed', 'dismissed'];
@@ -73,10 +95,10 @@ const ARCHIVE_ELIGIBLE_STATUSES = new Set(['completed', 'dismissed']);
// deleted out from under this process while it runs, and a flag-based cache
// would keep serving a since-deleted file's contents forever. `saveItems`
// INVALIDATES the cache after every write rather than seeding it from what it
-// wrote: two saveItems calls can interleave (a route and a cosEvents handler
-// — the same window as the documented lost-update race), and a post-write
-// stat could then pin one writer's content under the other writer's identity
-// and serve it until the next write. Invalidation keeps the invariant simple —
+// wrote: writes are serialized by `runMutation`, but unqueued reads (`getItems`,
+// `getPendingCounts`) can still overlap a write, and a post-write stat could
+// pin one writer's content under another turn's identity and serve it until
+// the next write. Invalidation keeps the invariant simple —
// the cache only ever holds content read from the file under its own identity
// — at the cost of one re-parse per write.
//
@@ -279,54 +301,56 @@ export async function createItem({ type, title, description = '', metadata = {}
throw err;
}
- const items = await loadItems();
-
- // Prevent duplicate alerts for same reference within 24 hours. Archive only
- // ever holds items >=30 days old, so it can never actually match this 24h
- // window in practice — but check it anyway rather than silently narrowing
- // the dedup window the day an item crosses into archive.json. Only
- // consulted when the live scan misses, so the common case (no dedup match,
- // or a live match) never touches the archive file. A corrupt archive.json
- // degrades to "no archived duplicate found" (`loadArchiveOrEmpty`) rather
- // than blocking item creation entirely.
- if (type === 'alert' && metadata?.referenceId) {
- const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000;
- const isDuplicate = (i) =>
- i.type === 'alert' &&
- i.metadata?.referenceId === metadata.referenceId &&
- new Date(i.createdAt).getTime() > oneDayAgo;
- const duplicate = items.find(isDuplicate) ?? (await loadArchiveOrEmpty('createItem duplicate check')).find(isDuplicate);
- if (duplicate) return duplicate;
- }
+ return runMutation(async () => {
+ const items = await loadItems();
+
+ // Prevent duplicate alerts for same reference within 24 hours. Archive only
+ // ever holds items >=30 days old, so it can never actually match this 24h
+ // window in practice — but check it anyway rather than silently narrowing
+ // the dedup window the day an item crosses into archive.json. Only
+ // consulted when the live scan misses, so the common case (no dedup match,
+ // or a live match) never touches the archive file. A corrupt archive.json
+ // degrades to "no archived duplicate found" (`loadArchiveOrEmpty`) rather
+ // than blocking item creation entirely.
+ if (type === 'alert' && metadata?.referenceId) {
+ const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000;
+ const isDuplicate = (i) =>
+ i.type === 'alert' &&
+ i.metadata?.referenceId === metadata.referenceId &&
+ new Date(i.createdAt).getTime() > oneDayAgo;
+ const duplicate = items.find(isDuplicate) ?? (await loadArchiveOrEmpty('createItem duplicate check')).find(isDuplicate);
+ if (duplicate) return { result: duplicate };
+ }
- const item = {
- id: uuidv4(),
- type,
- title,
- description,
- status: 'pending',
- metadata,
- createdAt: new Date().toISOString(),
- updatedAt: new Date().toISOString()
- };
+ const item = {
+ id: uuidv4(),
+ type,
+ title,
+ description,
+ status: 'pending',
+ metadata,
+ createdAt: new Date().toISOString(),
+ updatedAt: new Date().toISOString()
+ };
- items.push(item);
- await saveItems(items);
- console.log(`📋 Review item created: ${type} — ${title}`);
- reviewEvents.emit('item:created', item);
- return item;
+ items.push(item);
+ await saveItems(items);
+ console.log(`📋 Review item created: ${type} — ${title}`);
+ return { result: item, events: [['item:created', item]] };
+ });
}
-/**
- * Update an item's status
- */
-async function updateItemStatus(id, status, { allowSourceOwned = false } = {}) {
+function assertItemStatus(status) {
if (!ITEM_STATUSES.includes(status)) {
const err = new Error(`Invalid status: ${status}`);
err.status = 400;
throw err;
}
+}
+// Queue-free body: callers already inside `runMutation` (reopenItem) use this
+// directly; awaiting `updateItemStatus` from inside a region would deadlock.
+async function applyItemStatus(id, status, { allowSourceOwned = false } = {}) {
const { items, save } = await loadItemCollection(id);
const item = items.find(i => i.id === id);
if (!item) {
@@ -346,8 +370,15 @@ async function updateItemStatus(id, status, { allowSourceOwned = false } = {}) {
item.updatedAt = new Date().toISOString();
await save(items);
console.log(`📋 Review item ${status}: ${item.type} — ${item.title}`);
- reviewEvents.emit('item:updated', item);
- return item;
+ return { result: item, events: [['item:updated', item]] };
+}
+
+/**
+ * Update an item's status
+ */
+async function updateItemStatus(id, status, options) {
+ assertItemStatus(status);
+ return runMutation(() => applyItemStatus(id, status, options));
}
/**
@@ -363,26 +394,27 @@ export async function completeItem(id) {
* never remains stranded in cold storage.
*/
export async function reopenItem(id) {
- const liveItems = await loadItems();
- if (liveItems.some((item) => item.id === id)) {
- return updateItemStatus(id, 'pending');
- }
+ return runMutation(async () => {
+ const liveItems = await loadItems();
+ if (liveItems.some((item) => item.id === id)) {
+ return applyItemStatus(id, 'pending');
+ }
- const archive = await loadArchive();
- const index = archive.findIndex((item) => item.id === id);
- if (index === -1) {
- const err = new Error(`Review item not found: ${id}`);
- err.status = 404;
- throw err;
- }
+ const archive = await loadArchive();
+ const index = archive.findIndex((item) => item.id === id);
+ if (index === -1) {
+ const err = new Error(`Review item not found: ${id}`);
+ err.status = 404;
+ throw err;
+ }
- const [item] = archive.splice(index, 1);
- const reopened = { ...item, status: 'pending', updatedAt: new Date().toISOString() };
- await saveItems([...liveItems, reopened]);
- await atomicWrite(ARCHIVE_FILE, archive);
- console.log(`📋 Review item reopened: ${reopened.type} — ${reopened.title}`);
- reviewEvents.emit('item:updated', reopened);
- return reopened;
+ const [item] = archive.splice(index, 1);
+ const reopened = { ...item, status: 'pending', updatedAt: new Date().toISOString() };
+ await saveItems([...liveItems, reopened]);
+ await atomicWrite(ARCHIVE_FILE, archive);
+ console.log(`📋 Review item reopened: ${reopened.type} — ${reopened.title}`);
+ return { result: reopened, events: [['item:updated', reopened]] };
+ });
}
/**
@@ -394,8 +426,8 @@ export async function dismissItem(id) {
/**
* Bulk-update many items to the same status in a single read-modify-write.
- * Concurrent per-item POSTs race on saveItems and silently drop updates;
- * this endpoint handles the "Complete All" / "Dismiss All" cases atomically.
+ * One write (and one event) for the "Complete All" / "Dismiss All" cases
+ * instead of N per-item round trips; serialized with every other mutator.
* Pass `ids` to target specific items, or omit to target every pending item.
* Emits ONE `items:bulk-updated` event carrying every affected id rather than
* one `item:updated` per item — "Mark all read" over N pending items used to
@@ -403,77 +435,79 @@ export async function dismissItem(id) {
* every connected dashboard client.
*/
export async function bulkUpdateStatus({ ids, status }) {
- if (!ITEM_STATUSES.includes(status)) {
- const err = new Error(`Invalid status: ${status}`);
- err.status = 400;
- throw err;
- }
-
- const items = await loadItems();
- const idSet = Array.isArray(ids) && ids.length > 0 ? new Set(ids) : null;
- const updated = [];
- const now = new Date().toISOString();
- for (const item of items) {
- if (item.status !== 'pending') continue;
- if (idSet && !idSet.has(item.id)) continue;
- if (status === 'completed' && isSourceOwnedReviewItem(item)) continue;
- item.status = status;
- item.updatedAt = now;
- updated.push(item);
- }
+ assertItemStatus(status);
+
+ return runMutation(async () => {
+ const items = await loadItems();
+ const idSet = Array.isArray(ids) && ids.length > 0 ? new Set(ids) : null;
+ const updated = [];
+ const now = new Date().toISOString();
+ for (const item of items) {
+ if (item.status !== 'pending') continue;
+ if (idSet && !idSet.has(item.id)) continue;
+ if (status === 'completed' && isSourceOwnedReviewItem(item)) continue;
+ item.status = status;
+ item.updatedAt = now;
+ updated.push(item);
+ }
- if (updated.length === 0) return [];
+ if (updated.length === 0) return { result: [] };
- await saveItems(items);
- console.log(`📋 Review items bulk-${status}: ${updated.length}`);
- reviewEvents.emit('items:bulk-updated', { ids: updated.map(i => i.id), status, updatedAt: now });
- return updated;
+ await saveItems(items);
+ console.log(`📋 Review items bulk-${status}: ${updated.length}`);
+ return {
+ result: updated,
+ events: [['items:bulk-updated', { ids: updated.map(i => i.id), status, updatedAt: now }]]
+ };
+ });
}
/**
* Update an item's title and/or description
*/
export async function updateItem(id, { title, description }) {
- const { items, save } = await loadItemCollection(id);
- const item = items.find(i => i.id === id);
- if (!item) {
- const err = new Error(`Review item not found: ${id}`);
- err.status = 404;
- throw err;
- }
+ return runMutation(async () => {
+ const { items, save } = await loadItemCollection(id);
+ const item = items.find(i => i.id === id);
+ if (!item) {
+ const err = new Error(`Review item not found: ${id}`);
+ err.status = 404;
+ throw err;
+ }
- if (title !== undefined) item.title = title;
- if (description !== undefined) item.description = description;
- item.updatedAt = new Date().toISOString();
- await save(items);
- reviewEvents.emit('item:updated', item);
- return item;
+ if (title !== undefined) item.title = title;
+ if (description !== undefined) item.description = description;
+ item.updatedAt = new Date().toISOString();
+ await save(items);
+ return { result: item, events: [['item:updated', item]] };
+ });
}
/**
* Delete a review item
*/
export async function deleteItem(id) {
- const { items, save } = await loadItemCollection(id);
- const index = items.findIndex(i => i.id === id);
- if (index === -1) {
- const err = new Error(`Review item not found: ${id}`);
- err.status = 404;
- throw err;
- }
+ return runMutation(async () => {
+ const { items, save } = await loadItemCollection(id);
+ const index = items.findIndex(i => i.id === id);
+ if (index === -1) {
+ const err = new Error(`Review item not found: ${id}`);
+ err.status = 404;
+ throw err;
+ }
- if (isSourceOwnedReviewItem(items[index])) {
- const err = new Error('Source-owned review obligations require triage or their owning action');
- err.status = 409;
- err.code = 'SOURCE_ACTION_REQUIRED';
- throw err;
- }
+ if (isSourceOwnedReviewItem(items[index])) {
+ const err = new Error('Source-owned review obligations require triage or their owning action');
+ err.status = 409;
+ err.code = 'SOURCE_ACTION_REQUIRED';
+ throw err;
+ }
- const [removed] = items.splice(index, 1);
- await save(items);
- console.log(`📋 Review item deleted: ${removed.type} — ${removed.title}`);
- reviewEvents.emit('item:deleted', removed);
- return removed;
+ const [removed] = items.splice(index, 1);
+ await save(items);
+ console.log(`📋 Review item deleted: ${removed.type} — ${removed.title}`);
+ return { result: removed, events: [['item:deleted', removed]] };
+ });
}
/**
@@ -571,23 +605,20 @@ cosEvents.on(GOAL_FIDELITY_HOLD_EVENT, (data) => {
});
async function updateStatusByReferenceId(referenceId, status) {
- if (!ITEM_STATUSES.includes(status)) {
- const err = new Error(`Invalid status: ${status}`);
- err.status = 400;
- throw err;
- }
-
- const items = await loadItems();
- const matching = items.filter(i => i.metadata?.referenceId === referenceId && i.status === 'pending');
- if (matching.length === 0) return [];
- const now = new Date().toISOString();
- for (const item of matching) {
- item.status = status;
- item.updatedAt = now;
- }
- await saveItems(items);
- for (const item of matching) reviewEvents.emit('item:updated', item);
- return matching;
+ assertItemStatus(status);
+
+ return runMutation(async () => {
+ const items = await loadItems();
+ const matching = items.filter(i => i.metadata?.referenceId === referenceId && i.status === 'pending');
+ if (matching.length === 0) return { result: [] };
+ const now = new Date().toISOString();
+ for (const item of matching) {
+ item.status = status;
+ item.updatedAt = now;
+ }
+ await saveItems(items);
+ return { result: matching, events: matching.map(item => ['item:updated', item]) };
+ });
}
export const dismissByReferenceId = (referenceId) => updateStatusByReferenceId(referenceId, 'dismissed');