Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop tolerant reads from overwriting twin meta, genome meta and the agent index (#12440)

[._.]/ Adam Eivy6dc9d88f18c7
diff --git a/server/jsonWritebackConventions.test.js b/server/jsonWritebackConventions.test.js
index de15980aad..10ef6f22d7 100644
--- a/server/jsonWritebackConventions.test.js
+++ b/server/jsonWritebackConventions.test.js
@@ -80,6 +80,37 @@ describe('syntax scanner regression contracts', () => {
expect(strict).toEqual(['B', 'C', 'E', 'I', 'K']);
});
+ it('treats a raw read whose bytes reach safeJSONParse as a non-strict read (#12440)', () => {
+ const source = `
+ import { tryReadFile as slurp, atomicWrite } from './fileUtils.js';
+ async function loadA() { const raw = await tryReadFile(A); return safeJSONParse(raw ?? '{}', {}); }
+ async function loadB() { return safeJSONParse(await readFile(B, 'utf-8'), {}); }
+ async function loadC() { const raw = await slurp(C).catch(() => null); return safeJSONParse(raw, null); }
+ async function loadD() { const { ok, value } = await tryReadFileStrict(D); return ok ? safeJSONParse(value, null) : null; }
+ async function loadE() { const text = await readFile(E, 'utf-8'); return text.split('\\n'); }
+ async function loadF() { const raw = await tryReadFile(F); return helper(() => safeJSONParse(raw, null)); }
+ function unrelated() { const raw = 1; return safeJSONParse(raw, null); }
+ async function loadG() { const raw = await tryReadFile(G); return raw; }
+ async function loadH() { const raw = await tryReadFile(H); return [raw].map(function (raw) { return safeJSONParse(raw, null); }); }
+ async function loadJ() { const raw = await tryReadFile(J); { const raw = ''; } return safeJSONParse(raw, null); }
+ function parseK() { return safeJSONParse(raw, null); }
+ const raw = await tryReadFile(K);
+ async function loadL() { const raw = await tryReadFile(L); return safeJSONParse(other.raw, null); }
+ async function loadI() { const raw = await tryReadFile(I); return [1].map((raw) => safeJSONParse(raw, null)); }
+ await atomicWrite(A, 1); await atomicWrite(B, 1); await atomicWrite(C, 1); await atomicWrite(D, 1);
+ await atomicWrite(E, 1); await atomicWrite(F, 1); await atomicWrite(G, 1);
+ await atomicWrite(H, 1); await atomicWrite(I, 1); await atomicWrite(J, 1);
+ await atomicWrite(K, 1); await atomicWrite(L, 1);
+ `;
+ // A: variable, B: inline, C: aliased import, F: closure, J: an inner block's
+ // `raw` ends with the block, K: a closure declared before the binding it captures. D is a strict read,
+ // E, G and L (a `.raw` property) never reach a parse, and neither unrelated()'s `raw`
+ // nor a shadowing parameter (H, I) borrows the outer binding.
+ expect(scanJsonWriteback(source)).toEqual({ candidates: ['A', 'B', 'C', 'F', 'J', 'K'], strict: [] });
+ expect(mayReadJsonFile('tryReadFile(P); safeJSONParse(raw, {});')).toBe(true);
+ expect(scanJsonWriteback("const raw = await tryReadFile(P); safeJSONParse(raw, {}); atomicWrite(P, {});").candidates).toEqual(['P']);
+ });
+
it('keeps literal contents and dynamic template expressions distinct and fails on invalid syntax', () => {
expect(scanJsonWriteback('readJSONFile("a b", {}); atomicWrite("ab", {});').candidates).toEqual([]);
expect(scanJsonWriteback('readJSONFile(`row-${id}`, {}); atomicWrite(`row-${other}`, {});').candidates).toEqual([]);
diff --git a/server/services/cosAgentIndex.js b/server/services/cosAgentIndex.js
index f75942cca3..0023c73a93 100644
--- a/server/services/cosAgentIndex.js
+++ b/server/services/cosAgentIndex.js
@@ -15,7 +15,8 @@ import { readFile, writeFile, rename, readdir, rm, stat, cp } from 'fs/promises'
import { existsSync } from 'fs';
import { join } from 'path';
import { AGENTS_DIR, withStateLock } from './cosState.js';
-import { atomicWrite, ensureDir, safeJSONParse, tryReadFile } from '../lib/fileUtils.js';
+import { atomicWrite, ensureDir, safeJSONParse, tryReadFile, tryReadFileStrict } from '../lib/fileUtils.js';
+import { isPlainObject } from '../lib/objects.js';
import { repairCodexTaskSummary } from './codexSummaryRepair.js';
import {
loadCompletionOrderIndex,
@@ -48,11 +49,21 @@ async function loadAgentIndexLeased() {
await ensureDir(AGENTS_DIR);
}
- if (existsSync(INDEX_FILE)) {
- const content = await tryReadFile(INDEX_FILE);
- const parsed = safeJSONParse(content ?? '{}', {});
- agentIndex = new Map(Object.entries(parsed));
- console.log(`📂 Loaded agent index: ${agentIndex.size} entries`);
+ const { ok, value: content } = existsSync(INDEX_FILE)
+ ? await tryReadFileStrict(INDEX_FILE)
+ : { ok: true, value: null };
+ if (content !== null || !ok) {
+ const parsed = ok ? safeJSONParse(content, null, { allowArray: false }) : null;
+ if (isPlainObject(parsed)) {
+ agentIndex = new Map(Object.entries(parsed));
+ console.log(`📂 Loaded agent index: ${agentIndex.size} entries`);
+ } else {
+ // Present but unreadable or malformed is not an empty index: saving that
+ // would orphan every archived run. The date buckets are authoritative, so
+ // rebuild in memory; the next save persists the full map, never an empty one.
+ agentIndex = await scanDateBucketIndex();
+ console.error(`❌ Agent index is unreadable or malformed; rebuilt ${agentIndex.size} entries from date buckets: ${INDEX_FILE}`);
+ }
} else {
// No index yet — run migration from flat dirs to date buckets
agentIndex = await migrateAgentsToDateBuckets();
@@ -230,31 +241,42 @@ async function migrateAgentsToDateBuckets() {
return withBackupAssetPublication(migrateAgentsToDateBucketsLeased);
}
-async function migrateAgentsToDateBucketsLeased() {
+// Index every agent already archived under a YYYY-MM-DD bucket directory.
+async function scanDateBucketIndex(entries) {
const index = new Map();
-
- if (!existsSync(AGENTS_DIR)) {
- await ensureDir(AGENTS_DIR);
- await atomicWrite(INDEX_FILE, {});
- console.log('📂 Created empty agent index (no agents to migrate)');
- return index;
- }
-
- const entries = await readdir(AGENTS_DIR, { withFileTypes: true });
-
- // Also scan existing date-bucket dirs to include them in the index
+ const dirEntries = entries ?? await readdir(AGENTS_DIR, { withFileTypes: true });
const dateDirPattern = /^\d{4}-\d{2}-\d{2}$/;
- for (const entry of entries) {
+ for (const entry of dirEntries) {
if (!entry.isDirectory() || !dateDirPattern.test(entry.name)) continue;
const dateStr = entry.name;
const dateDir = join(AGENTS_DIR, dateStr);
- const agentDirs = await readdir(dateDir, { withFileTypes: true }).catch(() => []);
+ // A listing failure other than a vanished bucket must not read as an empty
+ // bucket: the caller may persist this map as the authoritative index.
+ const agentDirs = await readdir(dateDir, { withFileTypes: true }).catch(err => {
+ if (err.code === 'ENOENT') return [];
+ throw err;
+ });
for (const agentEntry of agentDirs) {
if (agentEntry.isDirectory() && agentEntry.name.startsWith('agent-')) {
index.set(agentEntry.name, dateStr);
}
}
}
+ return index;
+}
+
+async function migrateAgentsToDateBucketsLeased() {
+ if (!existsSync(AGENTS_DIR)) {
+ await ensureDir(AGENTS_DIR);
+ await atomicWrite(INDEX_FILE, {});
+ console.log('📂 Created empty agent index (no agents to migrate)');
+ return new Map();
+ }
+
+ const entries = await readdir(AGENTS_DIR, { withFileTypes: true });
+
+ // Also scan existing date-bucket dirs to include them in the index
+ const index = await scanDateBucketIndex(entries);
// Find flat agent-* dirs that need migration
const flatAgentDirs = entries.filter(e => e.isDirectory() && e.name.startsWith('agent-'));
diff --git a/server/services/digital-twin-meta.js b/server/services/digital-twin-meta.js
index a6bacfa541..75e5edf0d7 100644
--- a/server/services/digital-twin-meta.js
+++ b/server/services/digital-twin-meta.js
@@ -1,9 +1,10 @@
import { readFile, readdir } from 'fs/promises';
-import { existsSync } from 'fs';
import { join } from 'path';
import EventEmitter from 'events';
+import { AsyncLocalStorage } from 'node:async_hooks';
import { digitalTwinMetaSchema } from '../lib/digitalTwinValidation.js';
-import { safeJSONParse, atomicWrite } from '../lib/fileUtils.js';
+import { safeJSONParse, atomicWrite, tryReadFileStrict, unreadableStoreError } from '../lib/fileUtils.js';
+import { isPlainObject } from '../lib/objects.js';
import { createMutex } from '../lib/asyncMutex.js';
import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';
@@ -28,7 +29,12 @@ export const CACHE_TTL_MS = 5000;
// would otherwise each push onto it and the later saveMeta would persist
// whichever tombstone/record set it happened to hold. NOT re-entrant: never
// call withMetaLock from inside a withMetaLock callback.
-export const withMetaLock = createMutex();
+const metaMutex = createMutex();
+// Marks code running inside withMetaLock: loadMeta refuses an untrusted file
+// there instead of serving defaults, so a transient read fault cannot hand a
+// mutation defaults that a later, healthy read in saveMeta would let through.
+const metaLockContext = new AsyncLocalStorage();
+export const withMetaLock = (fn) => metaMutex(() => metaLockContext.run(true, fn));
// Expose cache for modules that manage test/document caches (testing.js)
export { cache };
@@ -59,15 +65,25 @@ export async function loadMeta() {
await ensureSoulDir();
- if (!existsSync(META_FILE)) {
- // Scan existing documents and build initial meta
+ const { ok, value: content } = await tryReadFileStrict(META_FILE);
+ if (ok && content === null) {
+ // Only ENOENT proves the file was never written: scan existing documents
+ // and build initial meta
const meta = await buildInitialMeta();
await saveMeta(meta);
return meta;
}
- const content = await readFile(META_FILE, 'utf-8');
- const parsed = safeJSONParse(content, DEFAULT_META);
+ const parsed = ok ? readMetaObject(content) : null;
+ if (!parsed) {
+ // Unreadable or malformed is not "no meta". Mutations (inside withMetaLock)
+ // refuse outright; readers get uncached defaults, and saveMeta still refuses
+ // to persist over the file until it is repaired, so the registry, histories
+ // and tombstones are never replaced.
+ console.error(`❌ Digital twin meta.json is unreadable or malformed: ${META_FILE}`);
+ if (metaLockContext.getStore()) throw unreadableStoreError(META_FILE);
+ return structuredClone(DEFAULT_META);
+ }
const validated = digitalTwinMetaSchema.safeParse(parsed);
cache.meta.data = validated.success ? validated.data : { ...DEFAULT_META, ...parsed };
@@ -165,8 +181,22 @@ function getPriorityForFile(filename) {
return priorities[filename] || 50;
}
+const readMetaObject = (content) => {
+ const parsed = safeJSONParse(content, null, { allowArray: false });
+ return isPlainObject(parsed) ? parsed : null;
+};
+
+// The on-disk file, not the caller's snapshot, decides whether a write is safe:
+// loadMeta serves defaults for an untrusted file, and any mutation built on
+// them would otherwise overwrite the real document registry and tombstones.
+async function assertMetaWritable() {
+ const { ok, value } = await tryReadFileStrict(META_FILE);
+ if (!ok || (value !== null && !readMetaObject(value))) throw unreadableStoreError(META_FILE);
+}
+
export async function saveMeta(meta) {
await ensureSoulDir();
+ await assertMetaWritable();
await atomicWrite(META_FILE, meta);
cache.meta.data = meta;
cache.meta.timestamp = Date.now();
diff --git a/server/services/digital-twin.test.js b/server/services/digital-twin.test.js
index 81fe007af3..66277dbead 100644
--- a/server/services/digital-twin.test.js
+++ b/server/services/digital-twin.test.js
@@ -42,6 +42,8 @@ vi.mock('../lib/digitalTwinValidation.js', async importOriginal => ({
vi.mock('../lib/fileUtils.js', () => ({
tryReadFile: vi.fn().mockResolvedValue(null),
+ tryReadFileStrict: vi.fn(),
+ unreadableStoreError: (filePath) => new Error(`Unreadable JSON file: ${filePath}`),
ensureDir: vi.fn(),
atomicWrite: vi.fn(),
safeJSONParse: vi.fn((str, defaultValue) => {
@@ -93,7 +95,7 @@ import { readFile, writeFile, unlink, readdir, mkdir, stat } from 'fs/promises';
import { existsSync } from 'fs';
import { getProviderById, getActiveProvider } from './providers.js';
import { buildPrompt } from './promptService.js';
-import { safeJSONParse, atomicWrite } from '../lib/fileUtils.js';
+import { safeJSONParse, atomicWrite, tryReadFileStrict } from '../lib/fileUtils.js';
import { runPromptThroughProvider } from './promptRunner.js';
import {
@@ -214,6 +216,16 @@ describe('digital-twin.js', () => {
// Default: directory exists, meta file exists
existsSync.mockReturnValue(true);
+ // A previous test's readFile stub must not leak in as the "on-disk" meta.json:
+ // saveMeta now reads the file back before replacing it.
+ readFile.mockReset();
+ // Strict reads honour the same fs mocks the suite already drives: a missing
+ // path (existsSync false) is ENOENT, anything else is the mocked readFile.
+ tryReadFileStrict.mockImplementation(async (filePath) => {
+ if (!existsSync(filePath)) return { ok: true, value: null };
+ const value = await readFile(filePath, 'utf8');
+ return { ok: true, value: value ?? null };
+ });
mkdir.mockResolvedValue(undefined);
writeFile.mockResolvedValue(undefined);
atomicWrite.mockResolvedValue(undefined);
diff --git a/server/services/genome.js b/server/services/genome.js
index 9fb0e52523..d1e4578233 100644
--- a/server/services/genome.js
+++ b/server/services/genome.js
@@ -1,7 +1,8 @@
import { noteReadinessChanged } from './readinessSignal.js';
import { join } from 'path';
import { randomUUID } from 'crypto';
-import { atomicWrite, PATHS, ensureDir, safeJSONParse, tryReadFile, unlinkGuarded } from '../lib/fileUtils.js';
+import { atomicWrite, PATHS, ensureDir, safeJSONParse, tryReadFile, tryReadFileStrict, unlinkGuarded, unreadableStoreError } from '../lib/fileUtils.js';
+import { isPlainObject } from '../lib/objects.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { CURATED_MARKERS, MARKER_CATEGORIES, classifyGenotype, formatGenotype, resolveApoeHaplotype } from '../lib/curatedGenomeMarkers.js';
@@ -28,15 +29,37 @@ async function ensureGenomeDir() {
await ensureDir(GENOME_DIR);
}
-async function loadMeta() {
+// Deep clone: a shallow spread would hand every caller DEFAULT_META.savedMarkers.
+const freshMeta = () => structuredClone(DEFAULT_META);
+
+const parseMeta = (raw) => {
+ const parsed = safeJSONParse(raw, null, { allowArray: false });
+ return isPlainObject(parsed) ? parsed : null;
+};
+
+// Only ENOENT is a genuinely absent file. An unreadable or malformed one is
+// served as defaults for display (`forWrite` false), but a mutation loads with
+// `forWrite` and refuses, and saveMeta refuses to persist over it, so saved
+// markers and the upload flag are never replaced by a failed read.
+async function loadMeta({ forWrite = false } = {}) {
await ensureGenomeDir();
- const raw = await tryReadFile(META_FILE);
- if (!raw) return { ...DEFAULT_META };
- return safeJSONParse(raw, { ...DEFAULT_META });
+ const { ok, value: raw } = await tryReadFileStrict(META_FILE);
+ if (ok && raw === null) return freshMeta();
+ const parsed = ok ? parseMeta(raw) : null;
+ if (parsed) return parsed;
+ console.error(`❌ Genome metadata is unreadable or malformed: ${META_FILE}`);
+ if (forWrite) throw unreadableStoreError(META_FILE);
+ return freshMeta();
+}
+
+async function assertMetaWritable() {
+ const { ok, value } = await tryReadFileStrict(META_FILE);
+ if (!ok || (value !== null && !parseMeta(value))) throw unreadableStoreError(META_FILE);
}
async function saveMeta(meta) {
await ensureGenomeDir();
+ await assertMetaWritable();
await atomicWrite(META_FILE, meta);
noteReadinessChanged();
}
@@ -118,6 +141,9 @@ export async function uploadGenome(content, filename) {
}
async function storeGenome(content, filename, index, build) {
+ // Refuse before replacing the raw file: the metadata that names it cannot be saved.
+ await assertMetaWritable();
+
// Save raw file
await atomicWrite(RAW_FILE, content);
@@ -126,7 +152,7 @@ async function storeGenome(content, filename, index, build) {
indexBuiltAt = Date.now();
// Save metadata
- const meta = await loadMeta();
+ const meta = await loadMeta({ forWrite: true });
meta.uploaded = true;
meta.uploadedAt = new Date().toISOString();
meta.filename = filename;
@@ -182,7 +208,7 @@ export async function scanCuratedMarkers() {
return { error: 'No genome data uploaded. Upload a genome file first.' };
}
- const meta = await loadMeta();
+ const meta = await loadMeta({ forWrite: true });
const results = [];
for (const marker of CURATED_MARKERS) {
@@ -296,7 +322,7 @@ export async function searchSNP(rsid) {
* Save a custom marker to the genome metadata.
*/
export async function saveMarker(data) {
- const meta = await loadMeta();
+ const meta = await loadMeta({ forWrite: true });
const id = randomUUID();
// Look up genotype from raw data if not provided
@@ -343,7 +369,7 @@ export async function saveMarker(data) {
* Update notes on a saved marker.
*/
export async function updateMarkerNotes(id, notes) {
- const meta = await loadMeta();
+ const meta = await loadMeta({ forWrite: true });
if (!meta.savedMarkers[id]) {
return { error: 'Marker not found' };
}
@@ -359,7 +385,7 @@ export async function updateMarkerNotes(id, notes) {
* Delete a saved marker.
*/
export async function deleteMarker(id) {
- const meta = await loadMeta();
+ const meta = await loadMeta({ forWrite: true });
if (!meta.savedMarkers[id]) {
return { error: 'Marker not found' };
}
diff --git a/server/services/lenientReadStoreSafety.test.js b/server/services/lenientReadStoreSafety.test.js
new file mode 100644
index 0000000000..a6e6904a25
--- /dev/null
+++ b/server/services/lenientReadStoreSafety.test.js
@@ -0,0 +1,165 @@
+/**
+ * Write-back boundaries for stores that used to read with `tryReadFile` +
+ * `safeJSONParse` (#12440): a truncated, empty, or unreadable file must never be
+ * replaced by defaults on the next mutation. Real disk under a temp data root;
+ * the EACCES case injects a read fault on the exact path.
+ */
+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdir, readFile, rm, writeFile } from 'fs/promises';
+import { dirname, join } from 'path';
+import { makePathsProxy } from '../lib/mockPathsDataRoot.js';
+
+const disk = vi.hoisted(() => {
+ const { mkdtempSync } = require('fs');
+ const { tmpdir } = require('os');
+ const { join } = require('path');
+ return { root: mkdtempSync(join(tmpdir(), 'lenient-read-store-safety-')), fault: null, once: false };
+});
+vi.mock('fs/promises', async (importOriginal) => {
+ const actual = await importOriginal();
+ return {
+ ...actual,
+ readFile: (...args) => {
+ if (args[0] === disk.fault) {
+ // `once` models a transient fault: later reads of the same path succeed.
+ if (disk.once) disk.fault = null;
+ return Promise.reject(Object.assign(new Error('injected read fault'), { code: 'EACCES' }));
+ }
+ return actual.readFile(...args);
+ }
+ };
+});
+vi.mock('../lib/fileUtils.js', async (importOriginal) =>
+ makePathsProxy(await importOriginal(), { dataRoot: disk.root, extraOverrides: { root: disk.root } }));
+vi.mock('./sharing/recordEvents.js', () => ({}));
+
+const { PATHS } = await import('../lib/fileUtils.js');
+
+afterEach(() => { disk.fault = null; disk.once = false; });
+afterAll(() => rm(disk.root, { recursive: true, force: true }));
+
+const unreadable = /Unreadable JSON file/;
+const write = async (path, bytes) => {
+ await mkdir(dirname(path), { recursive: true });
+ await writeFile(path, bytes);
+};
+
+describe('lenient-read stores refuse to write over an untrusted file', () => {
+ it('digital twin meta.json keeps its registry and tombstones, and still initializes when absent', async () => {
+ const twin = await import('./digital-twin-meta.js');
+ const path = twin.META_FILE;
+ const seed = JSON.stringify({ version: '1.0.0', documents: [{ id: 'keep' }], deletedDocuments: [{ id: 'gone' }], settings: {} });
+ for (const bytes of ['{truncated', '', '[]', seed]) {
+ twin.cache.meta.data = null;
+ await write(path, bytes);
+ disk.fault = bytes === seed ? path : null;
+ await expect(twin.updateSettings({ maxContextTokens: 1 })).rejects.toThrow(unreadable);
+ await expect(twin.saveMeta({ ...twin.DEFAULT_META })).rejects.toThrow(unreadable);
+ disk.fault = null;
+ expect(await readFile(path, 'utf8')).toBe(bytes);
+ }
+ // Repaired input recovers without a restart and keeps its unrelated records.
+ twin.cache.meta.data = null;
+ await write(path, seed);
+ await twin.updateSettings({ maxContextTokens: 1 });
+ const saved = JSON.parse(await readFile(path, 'utf8'));
+ expect(saved.deletedDocuments).toEqual([{ id: 'gone' }]);
+ expect(saved.settings.maxContextTokens).toBe(1);
+ // A transient fault on the load must not let defaults through when the
+ // guard's own re-read of the healthy file succeeds.
+ await write(path, seed);
+ twin.cache.meta.data = null;
+ disk.fault = path;
+ disk.once = true;
+ await expect(twin.updateSettings({ maxContextTokens: 3 })).rejects.toThrow(unreadable);
+ expect(JSON.parse(await readFile(path, 'utf8')).deletedDocuments).toEqual([{ id: 'gone' }]);
+ // The defaults served for an untrusted read are not the shared module object.
+ await write(path, '{truncated');
+ twin.cache.meta.data = null;
+ const served = await twin.loadMeta();
+ served.documents.push({ id: 'scratch' });
+ expect(twin.DEFAULT_META.documents).toEqual([]);
+ // Absent file initializes.
+ await rm(path);
+ twin.cache.meta.data = null;
+ await twin.updateSettings({ maxContextTokens: 2 });
+ expect(JSON.parse(await readFile(path, 'utf8')).settings.maxContextTokens).toBe(2);
+ });
+
+ it('genome metadata keeps saved markers and refuses an upload that cannot record itself', async () => {
+ const genome = await import('./genome.js');
+ const path = join(PATHS.meatspace, 'genome.json');
+ const rawPath = join(PATHS.meatspace, 'genome-raw.txt');
+ const seed = JSON.stringify({ uploaded: true, filename: 'mine.txt', savedMarkers: { rs1: { notes: 'keep' } } });
+ const tsv = Array.from({ length: 120 }, (_, i) => `rs${i + 1}\t1\t${i + 1}\tAA`).join('\n');
+ await write(rawPath, 'original raw bytes');
+ for (const bytes of ['{truncated', '', seed]) {
+ await write(path, bytes);
+ disk.fault = bytes === seed ? path : null;
+ await expect(genome.uploadGenome(tsv, 'new.txt')).rejects.toThrow(unreadable);
+ await expect(genome.saveMarker({ rsid: 'rs9', name: 'New', category: 'c', genotype: 'AA' })).rejects.toThrow(unreadable);
+ disk.fault = null;
+ expect(await readFile(path, 'utf8')).toBe(bytes);
+ expect(await readFile(rawPath, 'utf8')).toBe('original raw bytes');
+ }
+ await write(path, seed);
+ await genome.uploadGenome(tsv, 'new.txt');
+ const saved = JSON.parse(await readFile(path, 'utf8'));
+ expect(saved).toMatchObject({ uploaded: true, filename: 'new.txt', savedMarkers: { rs1: { notes: 'keep' } } });
+ disk.fault = path;
+ disk.once = true;
+ await expect(genome.saveMarker({ rsid: 'rs9', name: 'New', category: 'c', genotype: 'AA' })).rejects.toThrow(unreadable);
+ expect(JSON.parse(await readFile(path, 'utf8')).savedMarkers).toEqual({ rs1: { notes: 'keep' } });
+ await rm(path);
+ await genome.uploadGenome(tsv, 'fresh.txt');
+ expect(JSON.parse(await readFile(path, 'utf8')).filename).toBe('fresh.txt');
+ });
+
+ describe('CoS agent index', () => {
+ let index;
+ const agentsDir = () => join(PATHS.cos, 'agents');
+ const indexPath = () => join(agentsDir(), 'index.json');
+
+ beforeEach(async () => {
+ vi.resetModules();
+ await rm(agentsDir(), { recursive: true, force: true });
+ await mkdir(join(agentsDir(), '2026-10-01', 'agent-a'), { recursive: true });
+ await mkdir(join(agentsDir(), '2026-10-02', 'agent-b'), { recursive: true });
+ index = await import('./cosAgentIndex.js');
+ });
+
+ it('rebuilds a malformed index from the date buckets and never saves an empty map', async () => {
+ for (const bytes of ['{truncated', '', '[]']) {
+ vi.resetModules();
+ index = await import('./cosAgentIndex.js');
+ await write(indexPath(), bytes);
+ const loaded = await index.loadAgentIndex();
+ expect([...loaded.entries()].sort()).toEqual([['agent-a', '2026-10-01'], ['agent-b', '2026-10-02']]);
+ // Loading is read-only: the bytes stay until a mutation persists the rebuilt map.
+ expect(await readFile(indexPath(), 'utf8')).toBe(bytes);
+ expect(await index.addAgentArchivesToIndex([{ agentId: 'agent-c', date: '2026-10-03' }])).toBe(1);
+ expect(JSON.parse(await readFile(indexPath(), 'utf8'))).toEqual({
+ 'agent-a': '2026-10-01', 'agent-b': '2026-10-02', 'agent-c': '2026-10-03'
+ });
+ }
+ });
+
+ it('rebuilds instead of trusting an index it cannot read, and loads a healthy one verbatim', async () => {
+ const healthy = { 'agent-a': '2026-10-01', 'agent-z': '2026-09-30' };
+ await write(indexPath(), JSON.stringify(healthy));
+ disk.fault = indexPath();
+ expect([...(await index.loadAgentIndex()).keys()].sort()).toEqual(['agent-a', 'agent-b']);
+ disk.fault = null;
+ vi.resetModules();
+ index = await import('./cosAgentIndex.js');
+ expect(Object.fromEntries(await index.loadAgentIndex())).toEqual(healthy);
+ expect(JSON.parse(await readFile(indexPath(), 'utf8'))).toEqual(healthy);
+ });
+
+ it('still builds the index when none exists', async () => {
+ await rm(indexPath(), { force: true });
+ expect([...(await index.loadAgentIndex()).keys()].sort()).toEqual(['agent-a', 'agent-b']);
+ expect(Object.keys(JSON.parse(await readFile(indexPath(), 'utf8'))).sort()).toEqual(['agent-a', 'agent-b']);
+ });
+ });
+});
diff --git a/server/test/JSON_WRITEBACK.md b/server/test/JSON_WRITEBACK.md
index 3f47765c50..9cb62f1c34 100644
--- a/server/test/JSON_WRITEBACK.md
+++ b/server/test/JSON_WRITEBACK.md
@@ -12,6 +12,20 @@ Every exception records why a fallback cannot overwrite durable records. Review
that reason against the caller whenever the lifecycle changes. This is a review
inventory guard, not a proof that all exceptional control flow remains safe.
+## Raw reads feeding `safeJSONParse` (#12440)
+
+`tryReadFile` and `readFile` collapse "unreadable" into "absent" (or throw), and
+`safeJSONParse` turns malformed text into the caller's default. A raw read whose
+bytes reach `safeJSONParse`, through a variable in the same lexical scope (or an enclosing
+one; parameters and inner declarations shadow) or inline in its first argument, therefore counts as a non-strict
+read of that read's path and pairs with a same-path `atomicWrite` exactly like
+`readJSONFile`. `tryReadFileStrict` is strict and never counted. A raw read that
+never reaches a parse (a text file, a hash) is not a candidate. Convert a hit to
+`tryReadFileStrict` / `readJSONFileStrict` (ENOENT initializes; anything else
+refuses the write), or classify it in `jsonWritebackExceptions.js` with the reason
+the fallback cannot destroy durable data, or an `issue` that tracks the conversion.
+Worked examples: `digital-twin-meta.js`, `genome.js` and `cosAgentIndex.js`.
+
## Syntax and limits
Babel parses real call expressions, including nested arguments, options and
diff --git a/server/test/jsonWritebackExceptions.js b/server/test/jsonWritebackExceptions.js
index 80cd003d27..cef602df28 100644
--- a/server/test/jsonWritebackExceptions.js
+++ b/server/test/jsonWritebackExceptions.js
@@ -1,6 +1,11 @@
// Reviewed lifecycle exceptions from #6970–#6976. Keys are service-relative
// module + normalized first argument, never source positions. See JSON_WRITEBACK.md.
export const JSON_WRITEBACK_EXCEPTIONS = [
+ {
+ "key": "agentDrafts.js :: filePath",
+ "reason": "Tolerant read feeds saveDrafts: an unreadable drafts file reads as [] and the next save replaces it; conversion tracked in #12463.",
+ "issue": "#12463"
+ },
{
"key": "agentRunReconciler.js :: path",
"reason": "No failed-read write-back: repair planning and its final re-read skip null run metadata."
@@ -13,18 +18,65 @@ export const JSON_WRITEBACK_EXCEPTIONS = [
"key": "askConversations.js :: pathFor ( id )",
"reason": "No non-strict write-back: mutations use the strict default; only the listing sweep passes strict:false and skips unreadable members."
},
+ {
+ "key": "auth.js :: SESSIONS_FILE",
+ "reason": "Rebuildable: a lost session file only signs browsers out (login mints new sessions); it never grants access."
+ },
{
"key": "backup.js :: manifestPath",
"reason": "No write-back cycle: snapshot generation builds manifests from snapshot files; listing and restore only consume them."
},
+ {
+ "key": "browserService.js :: CONFIG_FILE",
+ "reason": "Tolerant read feeds saveConfig: a malformed config reads as defaults and the next save persists them; conversion tracked in #12463.",
+ "issue": "#12463"
+ },
{
"key": "claudeChangelog.js :: STATE_FILE",
"reason": "Cache-only: feed polling replaces the feed cache and its polling/seen cursors; there is no user-edit writer."
},
+ {
+ "key": "clinvar.js :: CLINVAR_INDEX",
+ "reason": "Cache-only: syncClinvar rebuilds the index and its metadata wholesale from the NCBI download; reads never feed a write."
+ },
+ {
+ "key": "codexSummaryRepair.js :: metaPath",
+ "reason": "No failed-read write-back: an unreadable or non-object metadata file returns null before the repair write."
+ },
+ {
+ "key": "contactsSync.js :: cachePath ( )",
+ "reason": "Cache-only: each sync replaces the whole contact cache from the source; the tolerant read only serves the UI."
+ },
+ {
+ "key": "contactsSync.js :: statePath ( )",
+ "reason": "Rebuildable status: a lost last-run record only resets the displayed run status; the next sync rewrites it."
+ },
+ {
+ "key": "cosAgentCompletionIndex.js :: orderFilePath ( )",
+ "reason": "Derived projection: an absent or corrupt file reads as empty and is backfilled from the archive metadata."
+ },
+ {
+ "key": "cosReports.js :: reportFile",
+ "reason": "No write-back cycle: reports are generated from agent records and written whole; getReport only projects a stored report."
+ },
+ {
+ "key": "feedbackLoop.js :: FEEDBACK_FILE",
+ "reason": "Tolerant read feeds saveFeedback: a malformed file reads as the empty default and the next entry overwrites it; conversion tracked in #12463.",
+ "issue": "#12463"
+ },
{
"key": "imageTo3d/sourceKeying.js :: preparedCacheMetadataPath ( targetPath )",
"reason": "Cache-only: source bytes and the explicit keying/framing request rebuild the prepared image metadata."
},
+ {
+ "key": "imessageManage.js :: blocklistPath ( )",
+ "reason": "Tolerant read feeds writeBlocklist: an unreadable blocklist reads as empty and the next edit drops every other handle; conversion tracked in #12463.",
+ "issue": "#12463"
+ },
+ {
+ "key": "imessageSync.js :: stateFilePath ( )",
+ "reason": "Rebuildable cursor: losing it re-reads recent messages, and the imsg:<guid> dedupe key makes the replay idempotent."
+ },
{
"key": "insightsService.js :: NARRATIVE_FILE",
"reason": "No non-strict write-back: display getter is tolerant; narrative refresh reads strictly before retaining prior text and timestamp."
@@ -33,15 +85,45 @@ export const JSON_WRITEBACK_EXCEPTIONS = [
"key": "insightsService.js :: THEMES_FILE",
"reason": "Authoritative rebuild: theme extraction replaces the cache from source evidence without retaining prior fields."
},
+ {
+ "key": "localModelAssessmentStore.js :: assessmentsFile ( )",
+ "reason": "Non-lossy: saveAssessment quarantines an unreadable store to a .corrupt-<ts> sibling before writing, and deleteAssessment refuses."
+ },
{
"key": "meatspaceDailyLog.js :: DAILY_LOG_FILE",
"reason": "No non-strict write-back: UI reads stay tolerant, but mutateDailyLog is the only writer and always reads with strict: true before atomicWrite.",
"issue": "#8032"
},
+ {
+ "key": "meatspacePostDrillCache.js :: CACHE_FILE",
+ "reason": "Cache-only: a lost drill cache costs one refill."
+ },
{
"key": "mediaSketches.js :: jsonPathFor ( key )",
"reason": "No write-back cycle: the read projects the canvas; explicit save replaces the complete canvas supplied by the user."
},
+ {
+ "key": "memoryClassifier.js :: MEMORY_CONFIG_FILE",
+ "reason": "Tolerant read feeds updateConfig: a malformed config reads as defaults and the next update persists them; conversion tracked in #12463.",
+ "issue": "#12463"
+ },
+ {
+ "key": "messagePlaywrightSync.js :: SELECTORS_FILE",
+ "reason": "Tolerant read feeds updateSelectors: malformed selectors read as {} and one provider's update replaces all; conversion tracked in #12463.",
+ "issue": "#12463"
+ },
+ {
+ "key": "messageSync.js :: filePath",
+ "reason": "Rebuildable cache: the per-account message cache and sync cursor are re-fetched from the provider; an unreadable cache reads as empty."
+ },
+ {
+ "key": "modelCapabilityTestStore.js :: storeFile ( )",
+ "reason": "Non-lossy: saveResult quarantines an unreadable store to a .corrupt-<ts> sibling before writing, and deleteResult refuses."
+ },
+ {
+ "key": "pm2.js :: dumpPath",
+ "reason": "No failed-read write-back: dropFromSavedList returns before the write when the dump is absent, unparseable, or not an array."
+ },
{
"key": "postRunStore.js :: SESSIONS_FILE",
"reason": "No non-strict write-back: loadFileSessions defaults strict=true and all mutation callers retain that default; tolerant reads cannot feed persistence."
@@ -54,6 +136,11 @@ export const JSON_WRITEBACK_EXCEPTIONS = [
"key": "providerQuotaShare.js :: PROVIDER_QUOTAS_FILE",
"reason": "No non-strict write-back: readLocalQuotaCards only projects UI cards; the quota merge writer reads strictly."
},
+ {
+ "key": "settings.js :: SETTINGS_FILE",
+ "reason": "Tolerant loadRaw serves display reads; partial writers must read strictly and refuse (fix in #12451, which also covers .env). Remove this entry if the scan stops reporting the pair.",
+ "issue": "#12451"
+ },
{
"key": "sharing/buckets.js :: bucketJsonPath",
"reason": "No non-strict write-back: the UI identity projection is tolerant; registration checks identity strictly before publishing."
@@ -66,6 +153,14 @@ export const JSON_WRITEBACK_EXCEPTIONS = [
"key": "sharing/manifest.js :: join ( bucketPath , \"manifests\" , filename )",
"reason": "No write-back cycle: transport manifests are input-only; exports build new manifests from authoritative records."
},
+ {
+ "key": "signalSync.js :: stateFilePath ( )",
+ "reason": "Rebuildable cursor: losing it re-reads recent messages, and the signal:<messageId> dedupe key makes the replay idempotent."
+ },
+ {
+ "key": "spotifySync.js :: stateFilePath ( )",
+ "reason": "Rebuildable cursor: losing it re-polls the recently-played window, and the spotify:<played_at>:<trackId> dedupe key makes the replay idempotent."
+ },
{
"key": "sprites/reference.js :: join ( candidatesDir , `${ name . replace ( /\\.png$/ , \"\" ) }.generation.json` )",
"reason": "No same-record write-back cycle: listing reads generation sidecars, generation writes new candidate names; approval reads provenance strictly."
@@ -75,6 +170,14 @@ export const JSON_WRITEBACK_EXCEPTIONS = [
"reason": "Authoritative rebuild: each probe writes a complete new evidence record from its own render; the tolerant read only feeds the status verdict and never a write.",
"issue": "#9412"
},
+ {
+ "key": "supersededLedger.js :: ledgerPath ( cosDir )",
+ "reason": "Rebuildable verdict cache: a missing or malformed ledger fails open to re-analysis of the affected branches."
+ },
+ {
+ "key": "taste-questionnaire.js :: TASTE_PROFILE_FILE",
+ "reason": "No non-strict write-back: the tolerant read serves display only; every mutation reads with readJSONFile strict:true via readProfileForMutation."
+ },
{
"key": "tools.js :: toolPath ( id )",
"reason": "No failed-read write-back: updateTool returns on null; registerTool is an explicit complete replacement."
@@ -94,5 +197,9 @@ export const JSON_WRITEBACK_EXCEPTIONS = [
{
"key": "weeklyDigest.js :: path",
"reason": "Authoritative rebuild: generation replaces the target week from agent records; reads display or compare a previous week, not a write-back base."
+ },
+ {
+ "key": "youtubeSync.js :: stateFilePath ( )",
+ "reason": "Rebuildable cursor: losing it re-reads recent history, and the yt:<videoId>:<localDay> dedupe key makes the replay idempotent."
}
];
diff --git a/server/test/jsonWritebackScan.js b/server/test/jsonWritebackScan.js
index ff71750440..793d2fc004 100644
--- a/server/test/jsonWritebackScan.js
+++ b/server/test/jsonWritebackScan.js
@@ -3,8 +3,9 @@ import { parse } from '@babel/parser';
// A cheap, conservative caller gate: aliases retain the imported helper name.
// Unicode escapes may spell an identifier without its literal name, so those
// sources still get parsed. Comments/strings are harmless false positives.
+// `safeJSONParse` admits the lenient raw-read shape (#12440).
export function mayReadJsonFile(source) {
- return source.includes('readJSONFile') || source.includes('\\u');
+ return source.includes('readJSONFile') || source.includes('safeJSONParse') || source.includes('\\u');
}
// Test-only syntax scan, never imports a service or reads runtime data.
@@ -33,6 +34,37 @@ function isStrict(options) {
return strict;
}
+const FUNCTION_NODES = new Set(['FunctionDeclaration', 'FunctionExpression', 'ArrowFunctionExpression', 'ObjectMethod', 'ClassMethod']);
+
+// Pre-order walk that also reports leaving a function, so callers can keep
+// per-function variable bindings (closures resolve through the stack).
+function walkScoped(node, enter, leave) {
+ if (!node || typeof node !== 'object') return;
+ if (Array.isArray(node)) { node.forEach(child => walkScoped(child, enter, leave)); return; }
+ if (node.type) enter(node);
+ for (const [key, child] of Object.entries(node)) {
+ if (!['loc', 'extra', 'tokens', 'comments'].includes(key)) walkScoped(child, enter, leave);
+ }
+ if (node.type) leave(node);
+}
+
+// Identifiers read as variables: not `.property` names or non-computed object keys.
+function referencedNames(node, names = []) {
+ if (!node || typeof node !== 'object') return names;
+ if (Array.isArray(node)) { node.forEach(child => referencedNames(child, names)); return names; }
+ if (node.type === 'Identifier') names.push(node.name);
+ else if (node.type === 'MemberExpression' || node.type === 'OptionalMemberExpression') {
+ referencedNames(node.object, names);
+ if (node.computed) referencedNames(node.property, names);
+ } else if (node.type === 'ObjectProperty' && !node.computed) referencedNames(node.value, names);
+ else {
+ for (const [key, child] of Object.entries(node)) {
+ if (!['loc', 'extra', 'tokens', 'comments'].includes(key)) referencedNames(child, names);
+ }
+ }
+ return names;
+}
+
function expressionKey(node, tokens, source) {
return tokens.filter(token => token.start >= node.start && token.end <= node.end
&& token.type.label && token.type.label !== 'eof')
@@ -43,7 +75,7 @@ function expressionKey(node, tokens, source) {
/** Same-module first-argument syntax equality; no control-flow proof implied. */
export function scanJsonWriteback(source) {
const ast = parse(source, { sourceType: 'module', tokens: true });
- const names = new Map(['readJSONFile', 'readJSONFileStrict', 'atomicWrite'].map(name => [name, name]));
+ const names = new Map(['readJSONFile', 'readJSONFileStrict', 'atomicWrite', 'tryReadFile', 'readFile', 'safeJSONParse'].map(name => [name, name]));
for (const statement of ast.program.body) {
if (statement.type !== 'ImportDeclaration') continue;
for (const specifier of statement.specifiers) {
@@ -55,15 +87,55 @@ export function scanJsonWriteback(source) {
const reads = new Set();
const strictReads = new Set();
const writes = new Set();
- walk(ast.program, node => {
- if (node.type !== 'CallExpression' || node.callee.type !== 'Identifier' || !node.arguments[0]) return;
- const name = names.get(node.callee.name);
- if (!name) return;
+ const calleeName = node => node.type === 'CallExpression' && node.callee.type === 'Identifier'
+ ? names.get(node.callee.name) : undefined;
+ const rawReadsIn = expression => {
+ const keys = [];
+ walk(expression, node => {
+ const name = calleeName(node);
+ if ((name === 'tryReadFile' || name === 'readFile') && node.arguments[0]) {
+ keys.push(expressionKey(node.arguments[0], ast.tokens, source));
+ }
+ });
+ return keys;
+ };
+ // `tryReadFile`/`readFile` collapse unreadable into absent (or throw) and
+ // `safeJSONParse` swallows malformed text, so a raw read whose bytes reach
+ // `safeJSONParse` through a variable or inline is a non-strict JSON read of
+ // that path. Bindings are lexical (function parameters, blocks, declarations
+ // shadow outer ones) and resolved after the walk so declaration order does not
+ // matter; other dataflow is out of scope.
+ const scopes = [new Map()];
+ const pending = [];
+ walkScoped(ast.program, node => {
+ if (FUNCTION_NODES.has(node.type) || node.type === 'BlockStatement') {
+ const scope = new Map();
+ if (FUNCTION_NODES.has(node.type)) walk(node.params, param => { if (param.type === 'Identifier') scope.set(param.name, []); });
+ scopes.push(scope);
+ }
+ if (node.type === 'VariableDeclarator' && node.id.type === 'Identifier') {
+ scopes.at(-1).set(node.id.name, node.init ? rawReadsIn(node.init) : []);
+ }
+ const name = calleeName(node);
+ if (!name || !node.arguments[0]) return;
+ if (name === 'safeJSONParse') {
+ for (const key of rawReadsIn(node.arguments[0])) reads.add(key);
+ pending.push({ names: referencedNames(node.arguments[0]), chain: [...scopes] });
+ return;
+ }
+ if (name === 'tryReadFile' || name === 'readFile') return;
const key = expressionKey(node.arguments[0], ast.tokens, source);
if (name === 'atomicWrite') writes.add(key);
else if (name === 'readJSONFileStrict' || isStrict(node.arguments[2])) strictReads.add(key);
else reads.add(key);
+ }, node => {
+ if (FUNCTION_NODES.has(node.type) || node.type === 'BlockStatement') scopes.pop();
});
+ for (const { names: referenced, chain } of pending) {
+ for (const identifier of referenced) {
+ chain.findLast(scope => scope.has(identifier))?.get(identifier).forEach(key => reads.add(key));
+ }
+ }
return {
candidates: [...reads].filter(key => writes.has(key)).sort(),
strict: [...strictReads].filter(key => writes.has(key)).sort(),