Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: serialize event-ledger history reads with rotation (#13182)

Adam Eivy4d6fb91cda4d
diff --git a/server/services/agentRunEventLog.js b/server/services/agentRunEventLog.js
index ee464b2fb4..3f9a7d80dc 100644
--- a/server/services/agentRunEventLog.js
+++ b/server/services/agentRunEventLog.js
@@ -448,19 +448,33 @@ export function flushRunEvents() {
}
/**
- * Queue a prune behind any in-flight appends.
+ * Run `fn` as one exclusive turn on the ledger queue, shared with appends,
+ * rotation, clearing and pruning.
*
- * The read paths call this so a ledger nobody is appending to still gets its
- * expired generations reclaimed. It goes through the SAME queue as appends —
- * `flushRunEvents()` alone would only prove the queue was empty a tick ago, not
- * that a rewrite can't land mid-append. Absorbs its own errors for the same
- * reason appends do: a read must never fail on account of housekeeping.
+ * Every read of a two-generation snapshot goes through here: rotation renames
+ * active over archive between two file reads, so a reader outside the queue can
+ * pair the old archive with the new active file and silently omit the
+ * generation that just rotated. Awaiting an empty queue first would not help —
+ * a writer can start before the reads finish.
+ *
+ * `fn` runs ON the queue, so it must never call another queued operation
+ * (appendRunEvent, clearPersistentMindHistory, withLedgerTurn): that would wait
+ * on a tail that is itself waiting for `fn`.
+ */
+function withLedgerTurn(fn) {
+ const turn = appendQueue.then(fn);
+ appendQueue = turn.then(() => undefined, () => undefined);
+ return turn;
+}
+
+/**
+ * Reclaim expired generations from inside a queued turn. Absorbs its own
+ * errors: a read must never fail on account of housekeeping.
*/
-function schedulePrune() {
- appendQueue = appendQueue.then(() => pruneExpired()).catch((err) => {
+async function pruneWithinTurn() {
+ await pruneExpired().catch((err) => {
console.error(`❌ Failed to prune CoS run event ledger: ${err.message}`);
});
- return appendQueue.then(() => undefined, () => undefined);
}
/**
@@ -480,23 +494,25 @@ function schedulePrune() {
* @param {number} [options.limit] - newest-N cap (default DEFAULT_READ_LIMIT)
* @returns {Promise<object[]>} validated events in append order
*/
-async function readRetainedEvents() {
- await hydrate();
- await schedulePrune();
- const [archive, active] = await Promise.all([
- readJSONLFile(ARCHIVE_PATH),
- readJSONLFile(ACTIVE_PATH),
- ]);
- // The age bound is enforced on read as well as on disk, so it holds in the
- // window between prunes — a reader must never see an event the retention
- // policy says is gone.
- const cutoff = expiryCutoff();
- // A line that fails the STRUCTURAL check is a corrupt/truncated write, not
- // data — dropping it keeps one bad line from poisoning the whole projection.
- // The check deliberately admits kinds this build does not know (see
- // `isStoredRunEvent`): a newer install's ledger must still read here. The
- // same predicate bounds the age, so a reader never sees an expired event.
- return [...archive, ...active].filter((event) => isRetained(event, cutoff) && !isMindShapedEvent(event));
+function readRetainedEvents() {
+ return withLedgerTurn(async () => {
+ await hydrate();
+ await pruneWithinTurn();
+ const [archive, active] = await Promise.all([
+ readJSONLFile(ARCHIVE_PATH),
+ readJSONLFile(ACTIVE_PATH),
+ ]);
+ // The age bound is enforced on read as well as on disk, so it holds in the
+ // window between prunes — a reader must never see an event the retention
+ // policy says is gone.
+ const cutoff = expiryCutoff();
+ // A line that fails the STRUCTURAL check is a corrupt/truncated write, not
+ // data — dropping it keeps one bad line from poisoning the whole projection.
+ // The check deliberately admits kinds this build does not know (see
+ // `isStoredRunEvent`): a newer install's ledger must still read here. The
+ // same predicate bounds the age, so a reader never sees an expired event.
+ return [...archive, ...active].filter((event) => isRetained(event, cutoff) && !isMindShapedEvent(event));
+ });
}
export async function readRunEvents({ runId, agentId, taskId, kind, since, limit } = {}) {
@@ -513,17 +529,19 @@ export async function readRunEvents({ runId, agentId, taskId, kind, since, limit
}
/** All retained events for one mind, ordered by durable sequence. */
-export async function readPersistentMindHistory(mindId = PERSISTENT_MIND_ID) {
- await hydrate();
- const [archive, active] = await Promise.all([
- readJSONLFile(MIND_ARCHIVE_PATH),
- readJSONLFile(MIND_ACTIVE_PATH),
- ]);
- return [...archive, ...active]
- // Preserve structurally valid future kinds in the physical mind stream.
- // Current projections can ignore an unknown kind without deleting history.
- .filter((event) => isMindShapedEvent(event) && event.mindId === mindId)
- .sort((a, b) => a.sequence - b.sequence || String(a.eventId).localeCompare(String(b.eventId)));
+export function readPersistentMindHistory(mindId = PERSISTENT_MIND_ID) {
+ return withLedgerTurn(async () => {
+ await hydrate();
+ const [archive, active] = await Promise.all([
+ readJSONLFile(MIND_ARCHIVE_PATH),
+ readJSONLFile(MIND_ACTIVE_PATH),
+ ]);
+ return [...archive, ...active]
+ // Preserve structurally valid future kinds in the physical mind stream.
+ // Current projections can ignore an unknown kind without deleting history.
+ .filter((event) => isMindShapedEvent(event) && event.mindId === mindId)
+ .sort((a, b) => a.sequence - b.sequence || String(a.eventId).localeCompare(String(b.eventId)));
+ });
}
/**
@@ -651,14 +669,16 @@ export async function getRunDiagnostic(id) {
* are held to, so "why is this run missing" has an answer that isn't a guess.
*/
export async function getRunEventLedgerStats() {
- await hydrate();
- await schedulePrune();
- const [archive, active, mindArchive, mindActive] = await Promise.all([
- readJSONLFile(ARCHIVE_PATH),
- readJSONLFile(ACTIVE_PATH),
- readJSONLFile(MIND_ARCHIVE_PATH),
- readJSONLFile(MIND_ACTIVE_PATH),
- ]);
+ const [archive, active, mindArchive, mindActive] = await withLedgerTurn(async () => {
+ await hydrate();
+ await pruneWithinTurn();
+ return Promise.all([
+ readJSONLFile(ARCHIVE_PATH),
+ readJSONLFile(ACTIVE_PATH),
+ readJSONLFile(MIND_ARCHIVE_PATH),
+ readJSONLFile(MIND_ACTIVE_PATH),
+ ]);
+ });
// Counted through the SAME predicate the read path uses, so "stats say 40
// events" and "the events endpoint returns 40" can never disagree.
const cutoff = expiryCutoff();
diff --git a/server/services/agentRunEventLog.test.js b/server/services/agentRunEventLog.test.js
index 600f82ea1d..52794d430c 100644
--- a/server/services/agentRunEventLog.test.js
+++ b/server/services/agentRunEventLog.test.js
@@ -14,9 +14,32 @@ const { LEDGER_DIR } = await vi.hoisted(async () => {
return { LEDGER_DIR: mk(j(tmp(), 'portos-run-events-')) };
});
+// Interleaving barriers: `beforeRead` runs ahead of every JSONL read, `afterRename`
+// after every rotation rename. Both are inert unless a test installs a hook.
+const barriers = vi.hoisted(() => ({ beforeRead: null, afterRename: null }));
+
vi.mock('../lib/fileUtils.js', async (importOriginal) => {
const actual = await importOriginal();
- return { ...actual, PATHS: { ...actual.PATHS, cos: LEDGER_DIR } };
+ return {
+ ...actual,
+ PATHS: { ...actual.PATHS, cos: LEDGER_DIR },
+ readJSONLFile: async (path, ...rest) => {
+ await barriers.beforeRead?.(path);
+ return actual.readJSONLFile(path, ...rest);
+ },
+ };
+});
+
+vi.mock('fs/promises', async (importOriginal) => {
+ const actual = await importOriginal();
+ return {
+ ...actual,
+ rename: async (...args) => {
+ const result = await actual.rename(...args);
+ barriers.afterRename?.();
+ return result;
+ },
+ };
});
const {
@@ -664,3 +687,83 @@ describe('retention — one predicate for reads, stats, and the prune (#4540)',
expect(await readRunEvents()).toHaveLength(1);
});
});
+
+describe('history reads versus generation rotation', () => {
+ const mindEvent = (index) => buildRunEvent({
+ kind: 'mind.wake',
+ mindId: 'cos-persistent-mind',
+ sequence: index + 1,
+ eventId: `mind-${index}`,
+ at: '2026-08-18T10:00:00.000Z',
+ data: { previousSequence: index || null },
+ });
+ const jsonl = (events) => events.map((event) => `${JSON.stringify(event)}\n`).join('');
+
+ // Seed one archived event and a FULL active generation, warm hydration, then
+ // hold a reader's active-file read while an append rotates the ledger. The
+ // append is not awaited before the gate opens: with the reader holding its
+ // queue turn the append cannot start, so the gate opens once rotation is seen
+ // or the append has demonstrably had every chance (event-loop turns, no timers).
+ async function readAcrossRotation(read) {
+ const archived = mindEvent(0);
+ const active = Array.from({ length: MAX_ACTIVE_MIND_EVENTS }, (_, index) => mindEvent(index + 1));
+ writeFileSync(MIND_ARCHIVE, jsonl([archived]));
+ writeFileSync(MIND_ACTIVE, jsonl(active));
+ await readPersistentMindHistory();
+
+ let openGate;
+ const gate = new Promise((resolve) => { openGate = resolve; });
+ let reachedActive;
+ const reached = new Promise((resolve) => { reachedActive = resolve; });
+ barriers.beforeRead = async (path) => {
+ if (path !== MIND_ACTIVE) return;
+ barriers.beforeRead = null;
+ reachedActive();
+ await gate;
+ };
+ let rotated;
+ const rotation = new Promise((resolve) => { rotated = resolve; });
+ barriers.afterRename = rotated;
+ try {
+ const reading = read({ archived, active });
+ await reached;
+ const appending = appendMindEvent({ kind: 'mind.wake', eventId: 'mind-new' });
+ let turns = 0;
+ const drained = (async () => { while (turns++ < 200) await new Promise((r) => setImmediate(r)); })();
+ await Promise.race([rotation, drained]);
+ openGate();
+ const [result] = await Promise.all([reading, appending]);
+ return { result, archived, active };
+ } finally {
+ barriers.beforeRead = null;
+ barriers.afterRename = null;
+ }
+ }
+
+ it('serves a complete pre-rotation history, never the old archive plus the new active file', async () => {
+ const { result, active } = await readAcrossRotation(() => readPersistentMindHistory());
+ expect(result).toHaveLength(active.length + 1);
+ expect(result.at(-1).eventId).toBe(active.at(-1).eventId);
+ // Rotation then drops the old archive, so the settled ledger is the rotated generation plus the new event.
+ const settled = await readPersistentMindHistory();
+ expect(settled).toHaveLength(active.length + 1);
+ expect(settled.at(-1).eventId).toBe('mind-new');
+ });
+
+ it('keeps trajectory cursor, hasMore and snapshot tied to one generation snapshot', async () => {
+ const { result, archived, active } = await readAcrossRotation(() => readPersistentMindEvents({
+ cursor: persistentMindEventCursor(mindEvent(0)),
+ limit: 500,
+ }));
+ expect(result.gap).toBe(false);
+ expect(result.hasMore).toBe(true);
+ expect(result.events).toHaveLength(500);
+ expect(result.events[0].eventId).toBe(active[0].eventId);
+ expect(archived.eventId).toBe('mind-0');
+ });
+
+ it('counts a complete generation pair in ledger stats while a rotation is racing it', async () => {
+ const { result, active } = await readAcrossRotation(() => getRunEventLedgerStats());
+ expect(result.mindArchivedEvents + result.mindActiveEvents).toBe(active.length + 1);
+ });
+});