Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(cos): coalesce task file reads to bound heap pressure (#12225)
Adam Eivy3da903850bec
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.jsindex 0511210d0f..5d9b551481 100644--- a/server/services/cosTaskStore.js+++ b/server/services/cosTaskStore.js@@ -20,6 +20,7 @@ import { DEVELOPMENT_ACTIVE_STATUSES, developmentWorkIdentity, sameDevelopmentWoimport { reviewerModelsFromDefaults } from '../lib/reviewerConfig.js';import { readFile, stat } from 'fs/promises';import { atomicWrite } from '../lib/fileUtils.js';+import { createSingleFlight } from '../lib/singleFlight.js';import { existsSync } from 'fs';import { join } from 'path';import { parseTasksMarkdown, groupTasksByStatus, getAutoApprovedTasks, getAwaitingApprovalTasks, generateTasksMarkdown, hasKnownPrefix, toRepresentableTask, PRIORITY_VALUES } from '../lib/taskParser.js';@@ -168,15 +169,18 @@ function isContentEdit(updates, existingMetadata = {}) {// Cache the parsed array per file so an unchanged file costs one `stat` plus a// structured clone instead of a read + full parse.//-// One server process, one user (AGENTS.md trust model) — no locking, no atomic-// write dance. Correctness rests on two independent invalidation signals:+// Concurrent API/daemon readers share the complete stat/read/parse operation.+// Otherwise a cold cache multiplies whole-file strings and parsed task arrays+// by the number of readers (#12225). Correctness rests on two signals://// 1. The stamp (`mtimeMs` + `size`) catches writes this module did NOT make —// a user editing `TASKS.md` in an editor, a restore, a peer's direct write.-// 2. Every write through `writeTaskFile` DROPS the entry outright. mtime can+// 2. Every write through `writeTaskFile` DROPS the entry before AND after the+// write, including reads started while the rename was pending. mtime can// be as coarse as one second on some filesystems, so a write-then-read in// the same tick must not depend on the stamp having moved.-const parsedTaskCache = new Map(); // filePath -> { stamp, tasks, byId? }+const parsedTaskCache = new Map(); // filePath -> generation { snapshot? }+const taskSnapshotReads = createSingleFlight();// `null` = could not stat (missing/unreadable) → do not cache, distinct from a// legitimately empty file, which stamps normally and caches its empty parse.@@ -188,14 +192,24 @@ const taskFileStamp = async (filePath) => {// Private snapshots never leave this module: list readers clone the array,// while ID lookups clone only the matching task. Both share invalidation.async function readTaskSnapshot(filePath) {- const stamp = await taskFileStamp(filePath);- const cached = parsedTaskCache.get(filePath);- if (stamp && cached?.stamp === stamp) return cached;-- const snapshot = { stamp, tasks: parseTasksMarkdown(await readFile(filePath, 'utf-8')) };- if (stamp) parsedTaskCache.set(filePath, snapshot);- else parsedTaskCache.delete(filePath);- return snapshot;+ let generation = parsedTaskCache.get(filePath);+ if (!generation) {+ generation = {};+ parsedTaskCache.set(filePath, generation);+ }+ // The entry identity is the flight key: invalidation detaches older reads+ // without letting them repopulate the cache or capture post-write callers.+ return taskSnapshotReads.run(generation, async () => {+ const stamp = await taskFileStamp(filePath);+ if (stamp && generation.snapshot?.stamp === stamp) return generation.snapshot;++ const snapshot = { stamp, tasks: parseTasksMarkdown(await readFile(filePath, 'utf-8')) };+ if (parsedTaskCache.get(filePath) === generation) {+ if (stamp) generation.snapshot = snapshot;+ else parsedTaskCache.delete(filePath);+ }+ return snapshot;+ });}async function readTaskFile(filePath) {@@ -226,7 +240,7 @@ async function writeTaskFile(filePath, markdown) {// task list, and a plain writeFile killed mid-write (PM2 restart, crash, full// disk) leaves it empty or half-written — which parses as "no tasks" and is// then persisted by the next write.- await atomicWrite(filePath, markdown);+ await atomicWrite(filePath, markdown).finally(() => parsedTaskCache.delete(filePath));}/** Test hook: forget every cached parse. */diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.jsindex 29a35a1a0a..c17624510b 100644--- a/server/services/cosTaskStore.test.js+++ b/server/services/cosTaskStore.test.js@@ -14,6 +14,8 @@ import { readFileSync as realReadFileSync } from 'node:fs';import { fileURLToPath } from 'url';import { dirname, join } from 'path';import { removeByMetadata } from './notifications.js';+import { readFile } from 'fs/promises';+import { atomicWrite } from '../lib/fileUtils.js';vi.mock('./prWatcher.js', () => ({ persistPrWatcherState: vi.fn(async () => ({})) }));import { persistPrWatcherState } from './prWatcher.js';@@ -398,6 +400,75 @@ describe('cosTaskStore parsed-task cache (#3497)', () => {expect(second.tasks).toEqual(first.tasks);});+ it('coalesces concurrent task views after cold and externally changed files', async () => {+ await addTask({ description: 'user task' }, 'user');+ await addTask({ description: 'internal task' }, 'internal');+ __resetTaskCache();+ mock.parseCalls = 0;+ vi.mocked(readFile).mockClear();++ const views = await Promise.all(Array.from({ length: 24 }, () => getTaskView({ kind: 'bare' })));+ expect(mock.parseCalls).toBe(2);+ expect(readFile).toHaveBeenCalledTimes(2);+ views[0].user.tasks[0].description = 'caller mutation';+ expect(views[1].user.tasks[0].description).toBe('user task');++ mock.files.set(USER_FILE, mock.files.get(USER_FILE).replace('user task', 'edited task'));+ mock.mtimes.set(USER_FILE, mock.mtimes.get(USER_FILE) + 1000);+ const changed = await Promise.all(Array.from({ length: 24 }, () => getTaskView({ kind: 'bare' })));+ expect(mock.parseCalls).toBe(3);+ expect(readFile).toHaveBeenCalledTimes(3);+ expect(changed.every(view => view.user.tasks[0].description === 'edited task')).toBe(true);+ });++ it('retries shared read failures on the next task view request', async () => {+ await addTask({ description: 'readable again' }, 'user');+ __resetTaskCache();+ vi.mocked(readFile).mockClear().mockRejectedValueOnce(new Error('temporary read failure'));+ const attempts = await Promise.allSettled(Array.from({ length: 8 }, () => getUserTasks()));+ expect(attempts.every(result => result.status === 'rejected' && result.reason.message === 'temporary read failure')).toBe(true);+ expect(readFile).toHaveBeenCalledTimes(1);+ expect((await getUserTasks()).tasks[0].description).toBe('readable again');+ expect(readFile).toHaveBeenCalledTimes(2);+ });++ it('keeps a read overlapping a write from capturing or poisoning post-write readers', async () => {+ const created = await addTask({ description: 'original' }, 'user');+ await getUserTasks();+ const originalContent = mock.files.get(USER_FILE);+ const originalMtime = mock.mtimes.get(USER_FILE);+ const writing = Promise.withResolvers();+ const finishWrite = Promise.withResolvers();+ const originalWrite = vi.mocked(atomicWrite).getMockImplementation();+ vi.mocked(atomicWrite).mockImplementationOnce(async (...args) => {+ writing.resolve();+ await finishWrite.promise;+ await originalWrite(...args);+ // A length-neutral edit on a coarse-mtime filesystem has the same stamp.+ mock.files.set(USER_FILE, originalContent.replace('original', 'REWRITTN'));+ mock.mtimes.set(USER_FILE, originalMtime);+ });+ const write = updateTask(created.id, { description: 'REWRITTN' }, 'user');+ await writing.promise;++ const reading = Promise.withResolvers();+ const finishRead = Promise.withResolvers();+ vi.mocked(readFile).mockImplementationOnce(async () => {+ reading.resolve();+ await finishRead.promise;+ return originalContent;+ });+ const overlappingRead = getUserTasks();+ await reading.promise;+ finishWrite.resolve();+ await write;+ // This must neither join the old flight nor consume its eventual snapshot.+ expect((await getUserTasks()).tasks[0].description).toBe('REWRITTN');+ finishRead.resolve();+ expect((await overlappingRead).tasks[0].description).toBe('original');+ expect((await getUserTasks()).tasks[0].description).toBe('REWRITTN');+ });+it('re-parses after an external edit bumps mtime', async () => {await addTask({ description: 'first' }, 'user');await getUserTasks();