Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: serialize event-ledger history reads with rotation (#13182)
Adam Eivy4d6fb91cda4d
diff --git a/server/services/agentRunEventLog.js b/server/services/agentRunEventLog.jsindex ee464b2fb4..3f9a7d80dc 100644--- a/server/services/agentRunEventLog.js+++ b/server/services/agentRunEventLog.js@@ -448,19 +448,33 @@ export function flushRunEvents() {}/**- * Queue a prune behind any in-flight appends.+ * Run `fn` as one exclusive turn on the ledger queue, shared with appends,+ * rotation, clearing and pruning.*- * The read paths call this so a ledger nobody is appending to still gets its- * expired generations reclaimed. It goes through the SAME queue as appends —- * `flushRunEvents()` alone would only prove the queue was empty a tick ago, not- * that a rewrite can't land mid-append. Absorbs its own errors for the same- * reason appends do: a read must never fail on account of housekeeping.+ * Every read of a two-generation snapshot goes through here: rotation renames+ * active over archive between two file reads, so a reader outside the queue can+ * pair the old archive with the new active file and silently omit the+ * generation that just rotated. Awaiting an empty queue first would not help —+ * a writer can start before the reads finish.+ *+ * `fn` runs ON the queue, so it must never call another queued operation+ * (appendRunEvent, clearPersistentMindHistory, withLedgerTurn): that would wait+ * on a tail that is itself waiting for `fn`.+ */+function withLedgerTurn(fn) {+ const turn = appendQueue.then(fn);+ appendQueue = turn.then(() => undefined, () => undefined);+ return turn;+}++/**+ * Reclaim expired generations from inside a queued turn. Absorbs its own+ * errors: a read must never fail on account of housekeeping.*/-function schedulePrune() {- appendQueue = appendQueue.then(() => pruneExpired()).catch((err) => {+async function pruneWithinTurn() {+ await pruneExpired().catch((err) => {console.error(`❌ Failed to prune CoS run event ledger: ${err.message}`);});- return appendQueue.then(() => undefined, () => undefined);}/**@@ -480,23 +494,25 @@ function schedulePrune() {* @param {number} [options.limit] - newest-N cap (default DEFAULT_READ_LIMIT)* @returns {Promise<object[]>} validated events in append order*/-async function readRetainedEvents() {- await hydrate();- await schedulePrune();- const [archive, active] = await Promise.all([- readJSONLFile(ARCHIVE_PATH),- readJSONLFile(ACTIVE_PATH),- ]);- // The age bound is enforced on read as well as on disk, so it holds in the- // window between prunes — a reader must never see an event the retention- // policy says is gone.- const cutoff = expiryCutoff();- // A line that fails the STRUCTURAL check is a corrupt/truncated write, not- // data — dropping it keeps one bad line from poisoning the whole projection.- // The check deliberately admits kinds this build does not know (see- // `isStoredRunEvent`): a newer install's ledger must still read here. The- // same predicate bounds the age, so a reader never sees an expired event.- return [...archive, ...active].filter((event) => isRetained(event, cutoff) && !isMindShapedEvent(event));+function readRetainedEvents() {+ return withLedgerTurn(async () => {+ await hydrate();+ await pruneWithinTurn();+ const [archive, active] = await Promise.all([+ readJSONLFile(ARCHIVE_PATH),+ readJSONLFile(ACTIVE_PATH),+ ]);+ // The age bound is enforced on read as well as on disk, so it holds in the+ // window between prunes — a reader must never see an event the retention+ // policy says is gone.+ const cutoff = expiryCutoff();+ // A line that fails the STRUCTURAL check is a corrupt/truncated write, not+ // data — dropping it keeps one bad line from poisoning the whole projection.+ // The check deliberately admits kinds this build does not know (see+ // `isStoredRunEvent`): a newer install's ledger must still read here. The+ // same predicate bounds the age, so a reader never sees an expired event.+ return [...archive, ...active].filter((event) => isRetained(event, cutoff) && !isMindShapedEvent(event));+ });}export async function readRunEvents({ runId, agentId, taskId, kind, since, limit } = {}) {@@ -513,17 +529,19 @@ export async function readRunEvents({ runId, agentId, taskId, kind, since, limit}/** All retained events for one mind, ordered by durable sequence. */-export async function readPersistentMindHistory(mindId = PERSISTENT_MIND_ID) {- await hydrate();- const [archive, active] = await Promise.all([- readJSONLFile(MIND_ARCHIVE_PATH),- readJSONLFile(MIND_ACTIVE_PATH),- ]);- return [...archive, ...active]- // Preserve structurally valid future kinds in the physical mind stream.- // Current projections can ignore an unknown kind without deleting history.- .filter((event) => isMindShapedEvent(event) && event.mindId === mindId)- .sort((a, b) => a.sequence - b.sequence || String(a.eventId).localeCompare(String(b.eventId)));+export function readPersistentMindHistory(mindId = PERSISTENT_MIND_ID) {+ return withLedgerTurn(async () => {+ await hydrate();+ const [archive, active] = await Promise.all([+ readJSONLFile(MIND_ARCHIVE_PATH),+ readJSONLFile(MIND_ACTIVE_PATH),+ ]);+ return [...archive, ...active]+ // Preserve structurally valid future kinds in the physical mind stream.+ // Current projections can ignore an unknown kind without deleting history.+ .filter((event) => isMindShapedEvent(event) && event.mindId === mindId)+ .sort((a, b) => a.sequence - b.sequence || String(a.eventId).localeCompare(String(b.eventId)));+ });}/**@@ -651,14 +669,16 @@ export async function getRunDiagnostic(id) {* are held to, so "why is this run missing" has an answer that isn't a guess.*/export async function getRunEventLedgerStats() {- await hydrate();- await schedulePrune();- const [archive, active, mindArchive, mindActive] = await Promise.all([- readJSONLFile(ARCHIVE_PATH),- readJSONLFile(ACTIVE_PATH),- readJSONLFile(MIND_ARCHIVE_PATH),- readJSONLFile(MIND_ACTIVE_PATH),- ]);+ const [archive, active, mindArchive, mindActive] = await withLedgerTurn(async () => {+ await hydrate();+ await pruneWithinTurn();+ return Promise.all([+ readJSONLFile(ARCHIVE_PATH),+ readJSONLFile(ACTIVE_PATH),+ readJSONLFile(MIND_ARCHIVE_PATH),+ readJSONLFile(MIND_ACTIVE_PATH),+ ]);+ });// Counted through the SAME predicate the read path uses, so "stats say 40// events" and "the events endpoint returns 40" can never disagree.const cutoff = expiryCutoff();diff --git a/server/services/agentRunEventLog.test.js b/server/services/agentRunEventLog.test.jsindex 600f82ea1d..52794d430c 100644--- a/server/services/agentRunEventLog.test.js+++ b/server/services/agentRunEventLog.test.js@@ -14,9 +14,32 @@ const { LEDGER_DIR } = await vi.hoisted(async () => {return { LEDGER_DIR: mk(j(tmp(), 'portos-run-events-')) };});+// Interleaving barriers: `beforeRead` runs ahead of every JSONL read, `afterRename`+// after every rotation rename. Both are inert unless a test installs a hook.+const barriers = vi.hoisted(() => ({ beforeRead: null, afterRename: null }));+vi.mock('../lib/fileUtils.js', async (importOriginal) => {const actual = await importOriginal();- return { ...actual, PATHS: { ...actual.PATHS, cos: LEDGER_DIR } };+ return {+ ...actual,+ PATHS: { ...actual.PATHS, cos: LEDGER_DIR },+ readJSONLFile: async (path, ...rest) => {+ await barriers.beforeRead?.(path);+ return actual.readJSONLFile(path, ...rest);+ },+ };+});++vi.mock('fs/promises', async (importOriginal) => {+ const actual = await importOriginal();+ return {+ ...actual,+ rename: async (...args) => {+ const result = await actual.rename(...args);+ barriers.afterRename?.();+ return result;+ },+ };});const {@@ -664,3 +687,83 @@ describe('retention — one predicate for reads, stats, and the prune (#4540)',expect(await readRunEvents()).toHaveLength(1);});});++describe('history reads versus generation rotation', () => {+ const mindEvent = (index) => buildRunEvent({+ kind: 'mind.wake',+ mindId: 'cos-persistent-mind',+ sequence: index + 1,+ eventId: `mind-${index}`,+ at: '2026-08-18T10:00:00.000Z',+ data: { previousSequence: index || null },+ });+ const jsonl = (events) => events.map((event) => `${JSON.stringify(event)}\n`).join('');++ // Seed one archived event and a FULL active generation, warm hydration, then+ // hold a reader's active-file read while an append rotates the ledger. The+ // append is not awaited before the gate opens: with the reader holding its+ // queue turn the append cannot start, so the gate opens once rotation is seen+ // or the append has demonstrably had every chance (event-loop turns, no timers).+ async function readAcrossRotation(read) {+ const archived = mindEvent(0);+ const active = Array.from({ length: MAX_ACTIVE_MIND_EVENTS }, (_, index) => mindEvent(index + 1));+ writeFileSync(MIND_ARCHIVE, jsonl([archived]));+ writeFileSync(MIND_ACTIVE, jsonl(active));+ await readPersistentMindHistory();++ let openGate;+ const gate = new Promise((resolve) => { openGate = resolve; });+ let reachedActive;+ const reached = new Promise((resolve) => { reachedActive = resolve; });+ barriers.beforeRead = async (path) => {+ if (path !== MIND_ACTIVE) return;+ barriers.beforeRead = null;+ reachedActive();+ await gate;+ };+ let rotated;+ const rotation = new Promise((resolve) => { rotated = resolve; });+ barriers.afterRename = rotated;+ try {+ const reading = read({ archived, active });+ await reached;+ const appending = appendMindEvent({ kind: 'mind.wake', eventId: 'mind-new' });+ let turns = 0;+ const drained = (async () => { while (turns++ < 200) await new Promise((r) => setImmediate(r)); })();+ await Promise.race([rotation, drained]);+ openGate();+ const [result] = await Promise.all([reading, appending]);+ return { result, archived, active };+ } finally {+ barriers.beforeRead = null;+ barriers.afterRename = null;+ }+ }++ it('serves a complete pre-rotation history, never the old archive plus the new active file', async () => {+ const { result, active } = await readAcrossRotation(() => readPersistentMindHistory());+ expect(result).toHaveLength(active.length + 1);+ expect(result.at(-1).eventId).toBe(active.at(-1).eventId);+ // Rotation then drops the old archive, so the settled ledger is the rotated generation plus the new event.+ const settled = await readPersistentMindHistory();+ expect(settled).toHaveLength(active.length + 1);+ expect(settled.at(-1).eventId).toBe('mind-new');+ });++ it('keeps trajectory cursor, hasMore and snapshot tied to one generation snapshot', async () => {+ const { result, archived, active } = await readAcrossRotation(() => readPersistentMindEvents({+ cursor: persistentMindEventCursor(mindEvent(0)),+ limit: 500,+ }));+ expect(result.gap).toBe(false);+ expect(result.hasMore).toBe(true);+ expect(result.events).toHaveLength(500);+ expect(result.events[0].eventId).toBe(active[0].eventId);+ expect(archived.eventId).toBe('mind-0');+ });++ it('counts a complete generation pair in ledger stats while a rotation is racing it', async () => {+ const { result, active } = await readAcrossRotation(() => getRunEventLedgerStats());+ expect(result.mindArchivedEvents + result.mindActiveEvents).toBe(active.length + 1);+ });+});