Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop tolerant reads from overwriting twin meta, genome meta and the agent index (#12440)
[._.]/ Adam Eivy6dc9d88f18c7
diff --git a/server/jsonWritebackConventions.test.js b/server/jsonWritebackConventions.test.jsindex de15980aad..10ef6f22d7 100644--- a/server/jsonWritebackConventions.test.js+++ b/server/jsonWritebackConventions.test.js@@ -80,6 +80,37 @@ describe('syntax scanner regression contracts', () => {expect(strict).toEqual(['B', 'C', 'E', 'I', 'K']);});+ it('treats a raw read whose bytes reach safeJSONParse as a non-strict read (#12440)', () => {+ const source = `+ import { tryReadFile as slurp, atomicWrite } from './fileUtils.js';+ async function loadA() { const raw = await tryReadFile(A); return safeJSONParse(raw ?? '{}', {}); }+ async function loadB() { return safeJSONParse(await readFile(B, 'utf-8'), {}); }+ async function loadC() { const raw = await slurp(C).catch(() => null); return safeJSONParse(raw, null); }+ async function loadD() { const { ok, value } = await tryReadFileStrict(D); return ok ? safeJSONParse(value, null) : null; }+ async function loadE() { const text = await readFile(E, 'utf-8'); return text.split('\\n'); }+ async function loadF() { const raw = await tryReadFile(F); return helper(() => safeJSONParse(raw, null)); }+ function unrelated() { const raw = 1; return safeJSONParse(raw, null); }+ async function loadG() { const raw = await tryReadFile(G); return raw; }+ async function loadH() { const raw = await tryReadFile(H); return [raw].map(function (raw) { return safeJSONParse(raw, null); }); }+ async function loadJ() { const raw = await tryReadFile(J); { const raw = ''; } return safeJSONParse(raw, null); }+ function parseK() { return safeJSONParse(raw, null); }+ const raw = await tryReadFile(K);+ async function loadL() { const raw = await tryReadFile(L); return safeJSONParse(other.raw, null); }+ async function loadI() { const raw = await tryReadFile(I); return [1].map((raw) => safeJSONParse(raw, null)); }+ await atomicWrite(A, 1); await atomicWrite(B, 1); await atomicWrite(C, 1); await atomicWrite(D, 1);+ await atomicWrite(E, 1); await atomicWrite(F, 1); await atomicWrite(G, 1);+ await atomicWrite(H, 1); await atomicWrite(I, 1); await atomicWrite(J, 1);+ await atomicWrite(K, 1); await atomicWrite(L, 1);+ `;+ // A: variable, B: inline, C: aliased import, F: closure, J: an inner block's+ // `raw` ends with the block, K: a closure declared before the binding it captures. D is a strict read,+ // E, G and L (a `.raw` property) never reach a parse, and neither unrelated()'s `raw`+ // nor a shadowing parameter (H, I) borrows the outer binding.+ expect(scanJsonWriteback(source)).toEqual({ candidates: ['A', 'B', 'C', 'F', 'J', 'K'], strict: [] });+ expect(mayReadJsonFile('tryReadFile(P); safeJSONParse(raw, {});')).toBe(true);+ expect(scanJsonWriteback("const raw = await tryReadFile(P); safeJSONParse(raw, {}); atomicWrite(P, {});").candidates).toEqual(['P']);+ });+it('keeps literal contents and dynamic template expressions distinct and fails on invalid syntax', () => {expect(scanJsonWriteback('readJSONFile("a b", {}); atomicWrite("ab", {});').candidates).toEqual([]);expect(scanJsonWriteback('readJSONFile(`row-${id}`, {}); atomicWrite(`row-${other}`, {});').candidates).toEqual([]);diff --git a/server/services/cosAgentIndex.js b/server/services/cosAgentIndex.jsindex f75942cca3..0023c73a93 100644--- a/server/services/cosAgentIndex.js+++ b/server/services/cosAgentIndex.js@@ -15,7 +15,8 @@ import { readFile, writeFile, rename, readdir, rm, stat, cp } from 'fs/promises'import { existsSync } from 'fs';import { join } from 'path';import { AGENTS_DIR, withStateLock } from './cosState.js';-import { atomicWrite, ensureDir, safeJSONParse, tryReadFile } from '../lib/fileUtils.js';+import { atomicWrite, ensureDir, safeJSONParse, tryReadFile, tryReadFileStrict } from '../lib/fileUtils.js';+import { isPlainObject } from '../lib/objects.js';import { repairCodexTaskSummary } from './codexSummaryRepair.js';import {loadCompletionOrderIndex,@@ -48,11 +49,21 @@ async function loadAgentIndexLeased() {await ensureDir(AGENTS_DIR);}- if (existsSync(INDEX_FILE)) {- const content = await tryReadFile(INDEX_FILE);- const parsed = safeJSONParse(content ?? '{}', {});- agentIndex = new Map(Object.entries(parsed));- console.log(`📂 Loaded agent index: ${agentIndex.size} entries`);+ const { ok, value: content } = existsSync(INDEX_FILE)+ ? await tryReadFileStrict(INDEX_FILE)+ : { ok: true, value: null };+ if (content !== null || !ok) {+ const parsed = ok ? safeJSONParse(content, null, { allowArray: false }) : null;+ if (isPlainObject(parsed)) {+ agentIndex = new Map(Object.entries(parsed));+ console.log(`📂 Loaded agent index: ${agentIndex.size} entries`);+ } else {+ // Present but unreadable or malformed is not an empty index: saving that+ // would orphan every archived run. The date buckets are authoritative, so+ // rebuild in memory; the next save persists the full map, never an empty one.+ agentIndex = await scanDateBucketIndex();+ console.error(`❌ Agent index is unreadable or malformed; rebuilt ${agentIndex.size} entries from date buckets: ${INDEX_FILE}`);+ }} else {// No index yet — run migration from flat dirs to date bucketsagentIndex = await migrateAgentsToDateBuckets();@@ -230,31 +241,42 @@ async function migrateAgentsToDateBuckets() {return withBackupAssetPublication(migrateAgentsToDateBucketsLeased);}-async function migrateAgentsToDateBucketsLeased() {+// Index every agent already archived under a YYYY-MM-DD bucket directory.+async function scanDateBucketIndex(entries) {const index = new Map();-- if (!existsSync(AGENTS_DIR)) {- await ensureDir(AGENTS_DIR);- await atomicWrite(INDEX_FILE, {});- console.log('📂 Created empty agent index (no agents to migrate)');- return index;- }-- const entries = await readdir(AGENTS_DIR, { withFileTypes: true });-- // Also scan existing date-bucket dirs to include them in the index+ const dirEntries = entries ?? await readdir(AGENTS_DIR, { withFileTypes: true });const dateDirPattern = /^\d{4}-\d{2}-\d{2}$/;- for (const entry of entries) {+ for (const entry of dirEntries) {if (!entry.isDirectory() || !dateDirPattern.test(entry.name)) continue;const dateStr = entry.name;const dateDir = join(AGENTS_DIR, dateStr);- const agentDirs = await readdir(dateDir, { withFileTypes: true }).catch(() => []);+ // A listing failure other than a vanished bucket must not read as an empty+ // bucket: the caller may persist this map as the authoritative index.+ const agentDirs = await readdir(dateDir, { withFileTypes: true }).catch(err => {+ if (err.code === 'ENOENT') return [];+ throw err;+ });for (const agentEntry of agentDirs) {if (agentEntry.isDirectory() && agentEntry.name.startsWith('agent-')) {index.set(agentEntry.name, dateStr);}}}+ return index;+}++async function migrateAgentsToDateBucketsLeased() {+ if (!existsSync(AGENTS_DIR)) {+ await ensureDir(AGENTS_DIR);+ await atomicWrite(INDEX_FILE, {});+ console.log('📂 Created empty agent index (no agents to migrate)');+ return new Map();+ }++ const entries = await readdir(AGENTS_DIR, { withFileTypes: true });++ // Also scan existing date-bucket dirs to include them in the index+ const index = await scanDateBucketIndex(entries);// Find flat agent-* dirs that need migrationconst flatAgentDirs = entries.filter(e => e.isDirectory() && e.name.startsWith('agent-'));diff --git a/server/services/digital-twin-meta.js b/server/services/digital-twin-meta.jsindex a6bacfa541..75e5edf0d7 100644--- a/server/services/digital-twin-meta.js+++ b/server/services/digital-twin-meta.js@@ -1,9 +1,10 @@import { readFile, readdir } from 'fs/promises';-import { existsSync } from 'fs';import { join } from 'path';import EventEmitter from 'events';+import { AsyncLocalStorage } from 'node:async_hooks';import { digitalTwinMetaSchema } from '../lib/digitalTwinValidation.js';-import { safeJSONParse, atomicWrite } from '../lib/fileUtils.js';+import { safeJSONParse, atomicWrite, tryReadFileStrict, unreadableStoreError } from '../lib/fileUtils.js';+import { isPlainObject } from '../lib/objects.js';import { createMutex } from '../lib/asyncMutex.js';import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';@@ -28,7 +29,12 @@ export const CACHE_TTL_MS = 5000;// would otherwise each push onto it and the later saveMeta would persist// whichever tombstone/record set it happened to hold. NOT re-entrant: never// call withMetaLock from inside a withMetaLock callback.-export const withMetaLock = createMutex();+const metaMutex = createMutex();+// Marks code running inside withMetaLock: loadMeta refuses an untrusted file+// there instead of serving defaults, so a transient read fault cannot hand a+// mutation defaults that a later, healthy read in saveMeta would let through.+const metaLockContext = new AsyncLocalStorage();+export const withMetaLock = (fn) => metaMutex(() => metaLockContext.run(true, fn));// Expose cache for modules that manage test/document caches (testing.js)export { cache };@@ -59,15 +65,25 @@ export async function loadMeta() {await ensureSoulDir();- if (!existsSync(META_FILE)) {- // Scan existing documents and build initial meta+ const { ok, value: content } = await tryReadFileStrict(META_FILE);+ if (ok && content === null) {+ // Only ENOENT proves the file was never written: scan existing documents+ // and build initial metaconst meta = await buildInitialMeta();await saveMeta(meta);return meta;}- const content = await readFile(META_FILE, 'utf-8');- const parsed = safeJSONParse(content, DEFAULT_META);+ const parsed = ok ? readMetaObject(content) : null;+ if (!parsed) {+ // Unreadable or malformed is not "no meta". Mutations (inside withMetaLock)+ // refuse outright; readers get uncached defaults, and saveMeta still refuses+ // to persist over the file until it is repaired, so the registry, histories+ // and tombstones are never replaced.+ console.error(`❌ Digital twin meta.json is unreadable or malformed: ${META_FILE}`);+ if (metaLockContext.getStore()) throw unreadableStoreError(META_FILE);+ return structuredClone(DEFAULT_META);+ }const validated = digitalTwinMetaSchema.safeParse(parsed);cache.meta.data = validated.success ? validated.data : { ...DEFAULT_META, ...parsed };@@ -165,8 +181,22 @@ function getPriorityForFile(filename) {return priorities[filename] || 50;}+const readMetaObject = (content) => {+ const parsed = safeJSONParse(content, null, { allowArray: false });+ return isPlainObject(parsed) ? parsed : null;+};++// The on-disk file, not the caller's snapshot, decides whether a write is safe:+// loadMeta serves defaults for an untrusted file, and any mutation built on+// them would otherwise overwrite the real document registry and tombstones.+async function assertMetaWritable() {+ const { ok, value } = await tryReadFileStrict(META_FILE);+ if (!ok || (value !== null && !readMetaObject(value))) throw unreadableStoreError(META_FILE);+}+export async function saveMeta(meta) {await ensureSoulDir();+ await assertMetaWritable();await atomicWrite(META_FILE, meta);cache.meta.data = meta;cache.meta.timestamp = Date.now();diff --git a/server/services/digital-twin.test.js b/server/services/digital-twin.test.jsindex 81fe007af3..66277dbead 100644--- a/server/services/digital-twin.test.js+++ b/server/services/digital-twin.test.js@@ -42,6 +42,8 @@ vi.mock('../lib/digitalTwinValidation.js', async importOriginal => ({vi.mock('../lib/fileUtils.js', () => ({tryReadFile: vi.fn().mockResolvedValue(null),+ tryReadFileStrict: vi.fn(),+ unreadableStoreError: (filePath) => new Error(`Unreadable JSON file: ${filePath}`),ensureDir: vi.fn(),atomicWrite: vi.fn(),safeJSONParse: vi.fn((str, defaultValue) => {@@ -93,7 +95,7 @@ import { readFile, writeFile, unlink, readdir, mkdir, stat } from 'fs/promises';import { existsSync } from 'fs';import { getProviderById, getActiveProvider } from './providers.js';import { buildPrompt } from './promptService.js';-import { safeJSONParse, atomicWrite } from '../lib/fileUtils.js';+import { safeJSONParse, atomicWrite, tryReadFileStrict } from '../lib/fileUtils.js';import { runPromptThroughProvider } from './promptRunner.js';import {@@ -214,6 +216,16 @@ describe('digital-twin.js', () => {// Default: directory exists, meta file existsexistsSync.mockReturnValue(true);+ // A previous test's readFile stub must not leak in as the "on-disk" meta.json:+ // saveMeta now reads the file back before replacing it.+ readFile.mockReset();+ // Strict reads honour the same fs mocks the suite already drives: a missing+ // path (existsSync false) is ENOENT, anything else is the mocked readFile.+ tryReadFileStrict.mockImplementation(async (filePath) => {+ if (!existsSync(filePath)) return { ok: true, value: null };+ const value = await readFile(filePath, 'utf8');+ return { ok: true, value: value ?? null };+ });mkdir.mockResolvedValue(undefined);writeFile.mockResolvedValue(undefined);atomicWrite.mockResolvedValue(undefined);diff --git a/server/services/genome.js b/server/services/genome.jsindex 9fb0e52523..d1e4578233 100644--- a/server/services/genome.js+++ b/server/services/genome.js@@ -1,7 +1,8 @@import { noteReadinessChanged } from './readinessSignal.js';import { join } from 'path';import { randomUUID } from 'crypto';-import { atomicWrite, PATHS, ensureDir, safeJSONParse, tryReadFile, unlinkGuarded } from '../lib/fileUtils.js';+import { atomicWrite, PATHS, ensureDir, safeJSONParse, tryReadFile, tryReadFileStrict, unlinkGuarded, unreadableStoreError } from '../lib/fileUtils.js';+import { isPlainObject } from '../lib/objects.js';import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';import { CURATED_MARKERS, MARKER_CATEGORIES, classifyGenotype, formatGenotype, resolveApoeHaplotype } from '../lib/curatedGenomeMarkers.js';@@ -28,15 +29,37 @@ async function ensureGenomeDir() {await ensureDir(GENOME_DIR);}-async function loadMeta() {+// Deep clone: a shallow spread would hand every caller DEFAULT_META.savedMarkers.+const freshMeta = () => structuredClone(DEFAULT_META);++const parseMeta = (raw) => {+ const parsed = safeJSONParse(raw, null, { allowArray: false });+ return isPlainObject(parsed) ? parsed : null;+};++// Only ENOENT is a genuinely absent file. An unreadable or malformed one is+// served as defaults for display (`forWrite` false), but a mutation loads with+// `forWrite` and refuses, and saveMeta refuses to persist over it, so saved+// markers and the upload flag are never replaced by a failed read.+async function loadMeta({ forWrite = false } = {}) {await ensureGenomeDir();- const raw = await tryReadFile(META_FILE);- if (!raw) return { ...DEFAULT_META };- return safeJSONParse(raw, { ...DEFAULT_META });+ const { ok, value: raw } = await tryReadFileStrict(META_FILE);+ if (ok && raw === null) return freshMeta();+ const parsed = ok ? parseMeta(raw) : null;+ if (parsed) return parsed;+ console.error(`❌ Genome metadata is unreadable or malformed: ${META_FILE}`);+ if (forWrite) throw unreadableStoreError(META_FILE);+ return freshMeta();+}++async function assertMetaWritable() {+ const { ok, value } = await tryReadFileStrict(META_FILE);+ if (!ok || (value !== null && !parseMeta(value))) throw unreadableStoreError(META_FILE);}async function saveMeta(meta) {await ensureGenomeDir();+ await assertMetaWritable();await atomicWrite(META_FILE, meta);noteReadinessChanged();}@@ -118,6 +141,9 @@ export async function uploadGenome(content, filename) {}async function storeGenome(content, filename, index, build) {+ // Refuse before replacing the raw file: the metadata that names it cannot be saved.+ await assertMetaWritable();+// Save raw fileawait atomicWrite(RAW_FILE, content);@@ -126,7 +152,7 @@ async function storeGenome(content, filename, index, build) {indexBuiltAt = Date.now();// Save metadata- const meta = await loadMeta();+ const meta = await loadMeta({ forWrite: true });meta.uploaded = true;meta.uploadedAt = new Date().toISOString();meta.filename = filename;@@ -182,7 +208,7 @@ export async function scanCuratedMarkers() {return { error: 'No genome data uploaded. Upload a genome file first.' };}- const meta = await loadMeta();+ const meta = await loadMeta({ forWrite: true });const results = [];for (const marker of CURATED_MARKERS) {@@ -296,7 +322,7 @@ export async function searchSNP(rsid) {* Save a custom marker to the genome metadata.*/export async function saveMarker(data) {- const meta = await loadMeta();+ const meta = await loadMeta({ forWrite: true });const id = randomUUID();// Look up genotype from raw data if not provided@@ -343,7 +369,7 @@ export async function saveMarker(data) {* Update notes on a saved marker.*/export async function updateMarkerNotes(id, notes) {- const meta = await loadMeta();+ const meta = await loadMeta({ forWrite: true });if (!meta.savedMarkers[id]) {return { error: 'Marker not found' };}@@ -359,7 +385,7 @@ export async function updateMarkerNotes(id, notes) {* Delete a saved marker.*/export async function deleteMarker(id) {- const meta = await loadMeta();+ const meta = await loadMeta({ forWrite: true });if (!meta.savedMarkers[id]) {return { error: 'Marker not found' };}diff --git a/server/services/lenientReadStoreSafety.test.js b/server/services/lenientReadStoreSafety.test.jsnew file mode 100644index 0000000000..a6e6904a25--- /dev/null+++ b/server/services/lenientReadStoreSafety.test.js@@ -0,0 +1,165 @@+/**+ * Write-back boundaries for stores that used to read with `tryReadFile` ++ * `safeJSONParse` (#12440): a truncated, empty, or unreadable file must never be+ * replaced by defaults on the next mutation. Real disk under a temp data root;+ * the EACCES case injects a read fault on the exact path.+ */+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { mkdir, readFile, rm, writeFile } from 'fs/promises';+import { dirname, join } from 'path';+import { makePathsProxy } from '../lib/mockPathsDataRoot.js';++const disk = vi.hoisted(() => {+ const { mkdtempSync } = require('fs');+ const { tmpdir } = require('os');+ const { join } = require('path');+ return { root: mkdtempSync(join(tmpdir(), 'lenient-read-store-safety-')), fault: null, once: false };+});+vi.mock('fs/promises', async (importOriginal) => {+ const actual = await importOriginal();+ return {+ ...actual,+ readFile: (...args) => {+ if (args[0] === disk.fault) {+ // `once` models a transient fault: later reads of the same path succeed.+ if (disk.once) disk.fault = null;+ return Promise.reject(Object.assign(new Error('injected read fault'), { code: 'EACCES' }));+ }+ return actual.readFile(...args);+ }+ };+});+vi.mock('../lib/fileUtils.js', async (importOriginal) =>+ makePathsProxy(await importOriginal(), { dataRoot: disk.root, extraOverrides: { root: disk.root } }));+vi.mock('./sharing/recordEvents.js', () => ({}));++const { PATHS } = await import('../lib/fileUtils.js');++afterEach(() => { disk.fault = null; disk.once = false; });+afterAll(() => rm(disk.root, { recursive: true, force: true }));++const unreadable = /Unreadable JSON file/;+const write = async (path, bytes) => {+ await mkdir(dirname(path), { recursive: true });+ await writeFile(path, bytes);+};++describe('lenient-read stores refuse to write over an untrusted file', () => {+ it('digital twin meta.json keeps its registry and tombstones, and still initializes when absent', async () => {+ const twin = await import('./digital-twin-meta.js');+ const path = twin.META_FILE;+ const seed = JSON.stringify({ version: '1.0.0', documents: [{ id: 'keep' }], deletedDocuments: [{ id: 'gone' }], settings: {} });+ for (const bytes of ['{truncated', '', '[]', seed]) {+ twin.cache.meta.data = null;+ await write(path, bytes);+ disk.fault = bytes === seed ? path : null;+ await expect(twin.updateSettings({ maxContextTokens: 1 })).rejects.toThrow(unreadable);+ await expect(twin.saveMeta({ ...twin.DEFAULT_META })).rejects.toThrow(unreadable);+ disk.fault = null;+ expect(await readFile(path, 'utf8')).toBe(bytes);+ }+ // Repaired input recovers without a restart and keeps its unrelated records.+ twin.cache.meta.data = null;+ await write(path, seed);+ await twin.updateSettings({ maxContextTokens: 1 });+ const saved = JSON.parse(await readFile(path, 'utf8'));+ expect(saved.deletedDocuments).toEqual([{ id: 'gone' }]);+ expect(saved.settings.maxContextTokens).toBe(1);+ // A transient fault on the load must not let defaults through when the+ // guard's own re-read of the healthy file succeeds.+ await write(path, seed);+ twin.cache.meta.data = null;+ disk.fault = path;+ disk.once = true;+ await expect(twin.updateSettings({ maxContextTokens: 3 })).rejects.toThrow(unreadable);+ expect(JSON.parse(await readFile(path, 'utf8')).deletedDocuments).toEqual([{ id: 'gone' }]);+ // The defaults served for an untrusted read are not the shared module object.+ await write(path, '{truncated');+ twin.cache.meta.data = null;+ const served = await twin.loadMeta();+ served.documents.push({ id: 'scratch' });+ expect(twin.DEFAULT_META.documents).toEqual([]);+ // Absent file initializes.+ await rm(path);+ twin.cache.meta.data = null;+ await twin.updateSettings({ maxContextTokens: 2 });+ expect(JSON.parse(await readFile(path, 'utf8')).settings.maxContextTokens).toBe(2);+ });++ it('genome metadata keeps saved markers and refuses an upload that cannot record itself', async () => {+ const genome = await import('./genome.js');+ const path = join(PATHS.meatspace, 'genome.json');+ const rawPath = join(PATHS.meatspace, 'genome-raw.txt');+ const seed = JSON.stringify({ uploaded: true, filename: 'mine.txt', savedMarkers: { rs1: { notes: 'keep' } } });+ const tsv = Array.from({ length: 120 }, (_, i) => `rs${i + 1}\t1\t${i + 1}\tAA`).join('\n');+ await write(rawPath, 'original raw bytes');+ for (const bytes of ['{truncated', '', seed]) {+ await write(path, bytes);+ disk.fault = bytes === seed ? path : null;+ await expect(genome.uploadGenome(tsv, 'new.txt')).rejects.toThrow(unreadable);+ await expect(genome.saveMarker({ rsid: 'rs9', name: 'New', category: 'c', genotype: 'AA' })).rejects.toThrow(unreadable);+ disk.fault = null;+ expect(await readFile(path, 'utf8')).toBe(bytes);+ expect(await readFile(rawPath, 'utf8')).toBe('original raw bytes');+ }+ await write(path, seed);+ await genome.uploadGenome(tsv, 'new.txt');+ const saved = JSON.parse(await readFile(path, 'utf8'));+ expect(saved).toMatchObject({ uploaded: true, filename: 'new.txt', savedMarkers: { rs1: { notes: 'keep' } } });+ disk.fault = path;+ disk.once = true;+ await expect(genome.saveMarker({ rsid: 'rs9', name: 'New', category: 'c', genotype: 'AA' })).rejects.toThrow(unreadable);+ expect(JSON.parse(await readFile(path, 'utf8')).savedMarkers).toEqual({ rs1: { notes: 'keep' } });+ await rm(path);+ await genome.uploadGenome(tsv, 'fresh.txt');+ expect(JSON.parse(await readFile(path, 'utf8')).filename).toBe('fresh.txt');+ });++ describe('CoS agent index', () => {+ let index;+ const agentsDir = () => join(PATHS.cos, 'agents');+ const indexPath = () => join(agentsDir(), 'index.json');++ beforeEach(async () => {+ vi.resetModules();+ await rm(agentsDir(), { recursive: true, force: true });+ await mkdir(join(agentsDir(), '2026-10-01', 'agent-a'), { recursive: true });+ await mkdir(join(agentsDir(), '2026-10-02', 'agent-b'), { recursive: true });+ index = await import('./cosAgentIndex.js');+ });++ it('rebuilds a malformed index from the date buckets and never saves an empty map', async () => {+ for (const bytes of ['{truncated', '', '[]']) {+ vi.resetModules();+ index = await import('./cosAgentIndex.js');+ await write(indexPath(), bytes);+ const loaded = await index.loadAgentIndex();+ expect([...loaded.entries()].sort()).toEqual([['agent-a', '2026-10-01'], ['agent-b', '2026-10-02']]);+ // Loading is read-only: the bytes stay until a mutation persists the rebuilt map.+ expect(await readFile(indexPath(), 'utf8')).toBe(bytes);+ expect(await index.addAgentArchivesToIndex([{ agentId: 'agent-c', date: '2026-10-03' }])).toBe(1);+ expect(JSON.parse(await readFile(indexPath(), 'utf8'))).toEqual({+ 'agent-a': '2026-10-01', 'agent-b': '2026-10-02', 'agent-c': '2026-10-03'+ });+ }+ });++ it('rebuilds instead of trusting an index it cannot read, and loads a healthy one verbatim', async () => {+ const healthy = { 'agent-a': '2026-10-01', 'agent-z': '2026-09-30' };+ await write(indexPath(), JSON.stringify(healthy));+ disk.fault = indexPath();+ expect([...(await index.loadAgentIndex()).keys()].sort()).toEqual(['agent-a', 'agent-b']);+ disk.fault = null;+ vi.resetModules();+ index = await import('./cosAgentIndex.js');+ expect(Object.fromEntries(await index.loadAgentIndex())).toEqual(healthy);+ expect(JSON.parse(await readFile(indexPath(), 'utf8'))).toEqual(healthy);+ });++ it('still builds the index when none exists', async () => {+ await rm(indexPath(), { force: true });+ expect([...(await index.loadAgentIndex()).keys()].sort()).toEqual(['agent-a', 'agent-b']);+ expect(Object.keys(JSON.parse(await readFile(indexPath(), 'utf8'))).sort()).toEqual(['agent-a', 'agent-b']);+ });+ });+});diff --git a/server/test/JSON_WRITEBACK.md b/server/test/JSON_WRITEBACK.mdindex 3f47765c50..9cb62f1c34 100644--- a/server/test/JSON_WRITEBACK.md+++ b/server/test/JSON_WRITEBACK.md@@ -12,6 +12,20 @@ Every exception records why a fallback cannot overwrite durable records. Reviewthat reason against the caller whenever the lifecycle changes. This is a reviewinventory guard, not a proof that all exceptional control flow remains safe.+## Raw reads feeding `safeJSONParse` (#12440)++`tryReadFile` and `readFile` collapse "unreadable" into "absent" (or throw), and+`safeJSONParse` turns malformed text into the caller's default. A raw read whose+bytes reach `safeJSONParse`, through a variable in the same lexical scope (or an enclosing+one; parameters and inner declarations shadow) or inline in its first argument, therefore counts as a non-strict+read of that read's path and pairs with a same-path `atomicWrite` exactly like+`readJSONFile`. `tryReadFileStrict` is strict and never counted. A raw read that+never reaches a parse (a text file, a hash) is not a candidate. Convert a hit to+`tryReadFileStrict` / `readJSONFileStrict` (ENOENT initializes; anything else+refuses the write), or classify it in `jsonWritebackExceptions.js` with the reason+the fallback cannot destroy durable data, or an `issue` that tracks the conversion.+Worked examples: `digital-twin-meta.js`, `genome.js` and `cosAgentIndex.js`.+## Syntax and limitsBabel parses real call expressions, including nested arguments, options anddiff --git a/server/test/jsonWritebackExceptions.js b/server/test/jsonWritebackExceptions.jsindex 80cd003d27..cef602df28 100644--- a/server/test/jsonWritebackExceptions.js+++ b/server/test/jsonWritebackExceptions.js@@ -1,6 +1,11 @@// Reviewed lifecycle exceptions from #6970–#6976. Keys are service-relative// module + normalized first argument, never source positions. See JSON_WRITEBACK.md.export const JSON_WRITEBACK_EXCEPTIONS = [+ {+ "key": "agentDrafts.js :: filePath",+ "reason": "Tolerant read feeds saveDrafts: an unreadable drafts file reads as [] and the next save replaces it; conversion tracked in #12463.",+ "issue": "#12463"+ },{"key": "agentRunReconciler.js :: path","reason": "No failed-read write-back: repair planning and its final re-read skip null run metadata."@@ -13,18 +18,65 @@ export const JSON_WRITEBACK_EXCEPTIONS = ["key": "askConversations.js :: pathFor ( id )","reason": "No non-strict write-back: mutations use the strict default; only the listing sweep passes strict:false and skips unreadable members."},+ {+ "key": "auth.js :: SESSIONS_FILE",+ "reason": "Rebuildable: a lost session file only signs browsers out (login mints new sessions); it never grants access."+ },{"key": "backup.js :: manifestPath","reason": "No write-back cycle: snapshot generation builds manifests from snapshot files; listing and restore only consume them."},+ {+ "key": "browserService.js :: CONFIG_FILE",+ "reason": "Tolerant read feeds saveConfig: a malformed config reads as defaults and the next save persists them; conversion tracked in #12463.",+ "issue": "#12463"+ },{"key": "claudeChangelog.js :: STATE_FILE","reason": "Cache-only: feed polling replaces the feed cache and its polling/seen cursors; there is no user-edit writer."},+ {+ "key": "clinvar.js :: CLINVAR_INDEX",+ "reason": "Cache-only: syncClinvar rebuilds the index and its metadata wholesale from the NCBI download; reads never feed a write."+ },+ {+ "key": "codexSummaryRepair.js :: metaPath",+ "reason": "No failed-read write-back: an unreadable or non-object metadata file returns null before the repair write."+ },+ {+ "key": "contactsSync.js :: cachePath ( )",+ "reason": "Cache-only: each sync replaces the whole contact cache from the source; the tolerant read only serves the UI."+ },+ {+ "key": "contactsSync.js :: statePath ( )",+ "reason": "Rebuildable status: a lost last-run record only resets the displayed run status; the next sync rewrites it."+ },+ {+ "key": "cosAgentCompletionIndex.js :: orderFilePath ( )",+ "reason": "Derived projection: an absent or corrupt file reads as empty and is backfilled from the archive metadata."+ },+ {+ "key": "cosReports.js :: reportFile",+ "reason": "No write-back cycle: reports are generated from agent records and written whole; getReport only projects a stored report."+ },+ {+ "key": "feedbackLoop.js :: FEEDBACK_FILE",+ "reason": "Tolerant read feeds saveFeedback: a malformed file reads as the empty default and the next entry overwrites it; conversion tracked in #12463.",+ "issue": "#12463"+ },{"key": "imageTo3d/sourceKeying.js :: preparedCacheMetadataPath ( targetPath )","reason": "Cache-only: source bytes and the explicit keying/framing request rebuild the prepared image metadata."},+ {+ "key": "imessageManage.js :: blocklistPath ( )",+ "reason": "Tolerant read feeds writeBlocklist: an unreadable blocklist reads as empty and the next edit drops every other handle; conversion tracked in #12463.",+ "issue": "#12463"+ },+ {+ "key": "imessageSync.js :: stateFilePath ( )",+ "reason": "Rebuildable cursor: losing it re-reads recent messages, and the imsg:<guid> dedupe key makes the replay idempotent."+ },{"key": "insightsService.js :: NARRATIVE_FILE","reason": "No non-strict write-back: display getter is tolerant; narrative refresh reads strictly before retaining prior text and timestamp."@@ -33,15 +85,45 @@ export const JSON_WRITEBACK_EXCEPTIONS = ["key": "insightsService.js :: THEMES_FILE","reason": "Authoritative rebuild: theme extraction replaces the cache from source evidence without retaining prior fields."},+ {+ "key": "localModelAssessmentStore.js :: assessmentsFile ( )",+ "reason": "Non-lossy: saveAssessment quarantines an unreadable store to a .corrupt-<ts> sibling before writing, and deleteAssessment refuses."+ },{"key": "meatspaceDailyLog.js :: DAILY_LOG_FILE","reason": "No non-strict write-back: UI reads stay tolerant, but mutateDailyLog is the only writer and always reads with strict: true before atomicWrite.","issue": "#8032"},+ {+ "key": "meatspacePostDrillCache.js :: CACHE_FILE",+ "reason": "Cache-only: a lost drill cache costs one refill."+ },{"key": "mediaSketches.js :: jsonPathFor ( key )","reason": "No write-back cycle: the read projects the canvas; explicit save replaces the complete canvas supplied by the user."},+ {+ "key": "memoryClassifier.js :: MEMORY_CONFIG_FILE",+ "reason": "Tolerant read feeds updateConfig: a malformed config reads as defaults and the next update persists them; conversion tracked in #12463.",+ "issue": "#12463"+ },+ {+ "key": "messagePlaywrightSync.js :: SELECTORS_FILE",+ "reason": "Tolerant read feeds updateSelectors: malformed selectors read as {} and one provider's update replaces all; conversion tracked in #12463.",+ "issue": "#12463"+ },+ {+ "key": "messageSync.js :: filePath",+ "reason": "Rebuildable cache: the per-account message cache and sync cursor are re-fetched from the provider; an unreadable cache reads as empty."+ },+ {+ "key": "modelCapabilityTestStore.js :: storeFile ( )",+ "reason": "Non-lossy: saveResult quarantines an unreadable store to a .corrupt-<ts> sibling before writing, and deleteResult refuses."+ },+ {+ "key": "pm2.js :: dumpPath",+ "reason": "No failed-read write-back: dropFromSavedList returns before the write when the dump is absent, unparseable, or not an array."+ },{"key": "postRunStore.js :: SESSIONS_FILE","reason": "No non-strict write-back: loadFileSessions defaults strict=true and all mutation callers retain that default; tolerant reads cannot feed persistence."@@ -54,6 +136,11 @@ export const JSON_WRITEBACK_EXCEPTIONS = ["key": "providerQuotaShare.js :: PROVIDER_QUOTAS_FILE","reason": "No non-strict write-back: readLocalQuotaCards only projects UI cards; the quota merge writer reads strictly."},+ {+ "key": "settings.js :: SETTINGS_FILE",+ "reason": "Tolerant loadRaw serves display reads; partial writers must read strictly and refuse (fix in #12451, which also covers .env). Remove this entry if the scan stops reporting the pair.",+ "issue": "#12451"+ },{"key": "sharing/buckets.js :: bucketJsonPath","reason": "No non-strict write-back: the UI identity projection is tolerant; registration checks identity strictly before publishing."@@ -66,6 +153,14 @@ export const JSON_WRITEBACK_EXCEPTIONS = ["key": "sharing/manifest.js :: join ( bucketPath , \"manifests\" , filename )","reason": "No write-back cycle: transport manifests are input-only; exports build new manifests from authoritative records."},+ {+ "key": "signalSync.js :: stateFilePath ( )",+ "reason": "Rebuildable cursor: losing it re-reads recent messages, and the signal:<messageId> dedupe key makes the replay idempotent."+ },+ {+ "key": "spotifySync.js :: stateFilePath ( )",+ "reason": "Rebuildable cursor: losing it re-polls the recently-played window, and the spotify:<played_at>:<trackId> dedupe key makes the replay idempotent."+ },{"key": "sprites/reference.js :: join ( candidatesDir , `${ name . replace ( /\\.png$/ , \"\" ) }.generation.json` )","reason": "No same-record write-back cycle: listing reads generation sidecars, generation writes new candidate names; approval reads provenance strictly."@@ -75,6 +170,14 @@ export const JSON_WRITEBACK_EXCEPTIONS = ["reason": "Authoritative rebuild: each probe writes a complete new evidence record from its own render; the tolerant read only feeds the status verdict and never a write.","issue": "#9412"},+ {+ "key": "supersededLedger.js :: ledgerPath ( cosDir )",+ "reason": "Rebuildable verdict cache: a missing or malformed ledger fails open to re-analysis of the affected branches."+ },+ {+ "key": "taste-questionnaire.js :: TASTE_PROFILE_FILE",+ "reason": "No non-strict write-back: the tolerant read serves display only; every mutation reads with readJSONFile strict:true via readProfileForMutation."+ },{"key": "tools.js :: toolPath ( id )","reason": "No failed-read write-back: updateTool returns on null; registerTool is an explicit complete replacement."@@ -94,5 +197,9 @@ export const JSON_WRITEBACK_EXCEPTIONS = [{"key": "weeklyDigest.js :: path","reason": "Authoritative rebuild: generation replaces the target week from agent records; reads display or compare a previous week, not a write-back base."+ },+ {+ "key": "youtubeSync.js :: stateFilePath ( )",+ "reason": "Rebuildable cursor: losing it re-reads recent history, and the yt:<videoId>:<localDay> dedupe key makes the replay idempotent."}];diff --git a/server/test/jsonWritebackScan.js b/server/test/jsonWritebackScan.jsindex ff71750440..793d2fc004 100644--- a/server/test/jsonWritebackScan.js+++ b/server/test/jsonWritebackScan.js@@ -3,8 +3,9 @@ import { parse } from '@babel/parser';// A cheap, conservative caller gate: aliases retain the imported helper name.// Unicode escapes may spell an identifier without its literal name, so those// sources still get parsed. Comments/strings are harmless false positives.+// `safeJSONParse` admits the lenient raw-read shape (#12440).export function mayReadJsonFile(source) {- return source.includes('readJSONFile') || source.includes('\\u');+ return source.includes('readJSONFile') || source.includes('safeJSONParse') || source.includes('\\u');}// Test-only syntax scan, never imports a service or reads runtime data.@@ -33,6 +34,37 @@ function isStrict(options) {return strict;}+const FUNCTION_NODES = new Set(['FunctionDeclaration', 'FunctionExpression', 'ArrowFunctionExpression', 'ObjectMethod', 'ClassMethod']);++// Pre-order walk that also reports leaving a function, so callers can keep+// per-function variable bindings (closures resolve through the stack).+function walkScoped(node, enter, leave) {+ if (!node || typeof node !== 'object') return;+ if (Array.isArray(node)) { node.forEach(child => walkScoped(child, enter, leave)); return; }+ if (node.type) enter(node);+ for (const [key, child] of Object.entries(node)) {+ if (!['loc', 'extra', 'tokens', 'comments'].includes(key)) walkScoped(child, enter, leave);+ }+ if (node.type) leave(node);+}++// Identifiers read as variables: not `.property` names or non-computed object keys.+function referencedNames(node, names = []) {+ if (!node || typeof node !== 'object') return names;+ if (Array.isArray(node)) { node.forEach(child => referencedNames(child, names)); return names; }+ if (node.type === 'Identifier') names.push(node.name);+ else if (node.type === 'MemberExpression' || node.type === 'OptionalMemberExpression') {+ referencedNames(node.object, names);+ if (node.computed) referencedNames(node.property, names);+ } else if (node.type === 'ObjectProperty' && !node.computed) referencedNames(node.value, names);+ else {+ for (const [key, child] of Object.entries(node)) {+ if (!['loc', 'extra', 'tokens', 'comments'].includes(key)) referencedNames(child, names);+ }+ }+ return names;+}+function expressionKey(node, tokens, source) {return tokens.filter(token => token.start >= node.start && token.end <= node.end&& token.type.label && token.type.label !== 'eof')@@ -43,7 +75,7 @@ function expressionKey(node, tokens, source) {/** Same-module first-argument syntax equality; no control-flow proof implied. */export function scanJsonWriteback(source) {const ast = parse(source, { sourceType: 'module', tokens: true });- const names = new Map(['readJSONFile', 'readJSONFileStrict', 'atomicWrite'].map(name => [name, name]));+ const names = new Map(['readJSONFile', 'readJSONFileStrict', 'atomicWrite', 'tryReadFile', 'readFile', 'safeJSONParse'].map(name => [name, name]));for (const statement of ast.program.body) {if (statement.type !== 'ImportDeclaration') continue;for (const specifier of statement.specifiers) {@@ -55,15 +87,55 @@ export function scanJsonWriteback(source) {const reads = new Set();const strictReads = new Set();const writes = new Set();- walk(ast.program, node => {- if (node.type !== 'CallExpression' || node.callee.type !== 'Identifier' || !node.arguments[0]) return;- const name = names.get(node.callee.name);- if (!name) return;+ const calleeName = node => node.type === 'CallExpression' && node.callee.type === 'Identifier'+ ? names.get(node.callee.name) : undefined;+ const rawReadsIn = expression => {+ const keys = [];+ walk(expression, node => {+ const name = calleeName(node);+ if ((name === 'tryReadFile' || name === 'readFile') && node.arguments[0]) {+ keys.push(expressionKey(node.arguments[0], ast.tokens, source));+ }+ });+ return keys;+ };+ // `tryReadFile`/`readFile` collapse unreadable into absent (or throw) and+ // `safeJSONParse` swallows malformed text, so a raw read whose bytes reach+ // `safeJSONParse` through a variable or inline is a non-strict JSON read of+ // that path. Bindings are lexical (function parameters, blocks, declarations+ // shadow outer ones) and resolved after the walk so declaration order does not+ // matter; other dataflow is out of scope.+ const scopes = [new Map()];+ const pending = [];+ walkScoped(ast.program, node => {+ if (FUNCTION_NODES.has(node.type) || node.type === 'BlockStatement') {+ const scope = new Map();+ if (FUNCTION_NODES.has(node.type)) walk(node.params, param => { if (param.type === 'Identifier') scope.set(param.name, []); });+ scopes.push(scope);+ }+ if (node.type === 'VariableDeclarator' && node.id.type === 'Identifier') {+ scopes.at(-1).set(node.id.name, node.init ? rawReadsIn(node.init) : []);+ }+ const name = calleeName(node);+ if (!name || !node.arguments[0]) return;+ if (name === 'safeJSONParse') {+ for (const key of rawReadsIn(node.arguments[0])) reads.add(key);+ pending.push({ names: referencedNames(node.arguments[0]), chain: [...scopes] });+ return;+ }+ if (name === 'tryReadFile' || name === 'readFile') return;const key = expressionKey(node.arguments[0], ast.tokens, source);if (name === 'atomicWrite') writes.add(key);else if (name === 'readJSONFileStrict' || isStrict(node.arguments[2])) strictReads.add(key);else reads.add(key);+ }, node => {+ if (FUNCTION_NODES.has(node.type) || node.type === 'BlockStatement') scopes.pop();});+ for (const { names: referenced, chain } of pending) {+ for (const identifier of referenced) {+ chain.findLast(scope => scope.has(identifier))?.get(identifier).forEach(key => reads.add(key));+ }+ }return {candidates: [...reads].filter(key => writes.has(key)).sort(),strict: [...strictReads].filter(key => writes.has(key)).sort(),