Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: refuse writes over unreadable drafts, browser, feedback, blocklist, classifier and selector stores (#12463)

[._.]/ Adam Eivy1fd116813a8b
diff --git a/server/services/agentDrafts.js b/server/services/agentDrafts.js
index 0f20012800..d51347560e 100644
--- a/server/services/agentDrafts.js
+++ b/server/services/agentDrafts.js
@@ -7,7 +7,7 @@
import { join } from 'path';
import { v4 as uuidv4 } from '../lib/uuid.js';
-import { atomicWrite, ensureDir, PATHS, safeJSONParse, tryReadFile } from '../lib/fileUtils.js';
+import { atomicWrite, ensureDir, PATHS, readJSONFileStrict, unreadableStoreError } from '../lib/fileUtils.js';
import { isTopLevelEntryName } from '../lib/pathSafety.js';
import { ServerError } from '../lib/errorHandler.js';
@@ -25,9 +25,10 @@ async function getDraftsPath(agentId) {
async function loadDrafts(agentId) {
const filePath = await getDraftsPath(agentId);
- const content = await tryReadFile(filePath);
- if (!content) return [];
- return safeJSONParse(content, [], { context: `agentDrafts:${agentId}` });
+ // Absent initializes to []; an unreadable file throws so no caller can save over it.
+ const { ok, value } = await readJSONFileStrict(filePath, []);
+ if (!ok) throw unreadableStoreError(filePath);
+ return value;
}
async function saveDrafts(agentId, drafts) {
diff --git a/server/services/agentDrafts.test.js b/server/services/agentDrafts.test.js
index 1f6eb84a20..7ddeb93c57 100644
--- a/server/services/agentDrafts.test.js
+++ b/server/services/agentDrafts.test.js
@@ -22,13 +22,16 @@ vi.mock('../lib/uuid.js', () => {
});
vi.mock('../lib/fileUtils.js', () => ({
- tryReadFile: vi.fn(async (path) => fileStore.has(path) ? fileStore.get(path) : null),
+ readJSONFileStrict: vi.fn(async (path, fallback) => {
+ if (!fileStore.has(path)) return { ok: true, value: fallback };
+ try { return { ok: true, value: JSON.parse(fileStore.get(path)) }; } catch { return { ok: false, value: fallback }; }
+ }),
+ unreadableStoreError: (path) => new Error(`Unreadable JSON file: ${path}`),
atomicWrite: vi.fn(async (path, data) => {
fileStore.set(path, typeof data === 'string' ? data : JSON.stringify(data, null, 2));
}),
ensureDir: vi.fn().mockResolvedValue(undefined),
- PATHS: { agentPersonalities: '/mock/agents' },
- safeJSONParse: (s, fallback) => { try { return JSON.parse(s); } catch { return fallback; } }
+ PATHS: { agentPersonalities: '/mock/agents' }
}));
const { listDrafts, getDraft, createDraft, updateDraft, deleteDraft } = await import('./agentDrafts.js');
diff --git a/server/services/browserService.js b/server/services/browserService.js
index ea98825dae..6765ed75a2 100644
--- a/server/services/browserService.js
+++ b/server/services/browserService.js
@@ -7,7 +7,7 @@
import { readdir, stat, unlink } from 'fs/promises';
import { join, basename, dirname, resolve, extname } from 'path';
import { EventEmitter } from 'events';
-import { ensureDir, safeJSONParse, PATHS, tryReadFile, atomicWrite, sleep } from '../lib/fileUtils.js';
+import { ensureDir, safeJSONParse, PATHS, readJSONFileStrict, unreadableStoreError, atomicWrite, sleep } from '../lib/fileUtils.js';
import { PORTS } from '../lib/ports.js';
import { normalizeBrowserConfig } from '../lib/browserConfig.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
@@ -58,15 +58,26 @@ async function getConfigMtimeMs() {
export async function loadConfig() {
const mtimeMs = await getConfigMtimeMs();
if (cachedConfig && cachedConfigMtimeMs === mtimeMs) return cachedConfig;
- const raw = await tryReadFile(CONFIG_FILE);
- const parsed = safeJSONParse(raw, null);
- cachedConfig = normalizeBrowserConfig(parsed ? { ...DEFAULT_CONFIG, ...parsed } : { ...DEFAULT_CONFIG });
+ const { ok, value: parsed } = await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false });
+ const config = normalizeBrowserConfig(parsed ? { ...DEFAULT_CONFIG, ...parsed } : { ...DEFAULT_CONFIG });
+ // Display readers serve defaults for an unreadable file, but never cache them:
+ // a repaired file must be picked up even when its mtime is unchanged.
+ if (!ok) return config;
+ cachedConfig = config;
cachedConfigMtimeMs = mtimeMs;
return cachedConfig;
}
+// Every write path refuses while the stored bytes are unreadable, so defaults
+// served to display readers are never persisted over the real config.
+async function assertConfigWritable() {
+ const { ok } = await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false });
+ if (!ok) throw unreadableStoreError(CONFIG_FILE);
+}
+
export async function saveConfig(config) {
await ensureDir(PATHS.data);
+ await assertConfigWritable();
cachedConfig = normalizeBrowserConfig({ ...DEFAULT_CONFIG, ...config });
await atomicWrite(CONFIG_FILE, cachedConfig);
cachedConfigMtimeMs = await getConfigMtimeMs();
diff --git a/server/services/imessageManage.js b/server/services/imessageManage.js
index ad600fe37b..e480601ef7 100644
--- a/server/services/imessageManage.js
+++ b/server/services/imessageManage.js
@@ -6,7 +6,7 @@
* remove rows from the machine-local `human_activity_events` table only.
* Blocklisted handles are skipped on future syncs (see imessageSync.runSync).
*/
-import { dataPath, atomicWrite, tryReadFile, safeJSONParse } from '../lib/fileUtils.js';
+import { dataPath, atomicWrite, readJSONFileStrict, unreadableStoreError } from '../lib/fileUtils.js';
import { identityFromHandle, normalizeIdentifier, normalizePhone } from '../lib/tribeMatch.js';
import * as humanActivity from './humanActivity.js';
import { getStatus } from './imessageSync.js';
@@ -61,9 +61,7 @@ export function blocklistKey(handle) {
return raw;
}
-export async function readBlocklist() {
- const raw = await tryReadFile(blocklistPath());
- const parsed = raw ? safeJSONParse(raw, null, { allowArray: false }) : null;
+function projectBlocklist(parsed) {
const handles = Array.isArray(parsed?.handles)
? [...new Set(parsed.handles.map((h) => blocklistKey(h)).filter(Boolean))].sort()
: [];
@@ -73,6 +71,19 @@ export async function readBlocklist() {
};
}
+// Display readers (status, conversation list) serve an empty list for an
+// unreadable file; mutation paths use readBlocklistForWrite and refuse.
+export async function readBlocklist() {
+ const { value } = await readJSONFileStrict(blocklistPath(), null, { allowArray: false });
+ return projectBlocklist(value);
+}
+
+async function readBlocklistForWrite() {
+ const { ok, value } = await readJSONFileStrict(blocklistPath(), null, { allowArray: false });
+ if (!ok) throw unreadableStoreError(blocklistPath());
+ return projectBlocklist(value);
+}
+
async function writeBlocklist(handles) {
const cleaned = [...new Set((handles || []).map((h) => blocklistKey(h)).filter(Boolean))].sort();
const payload = { handles: cleaned, updatedAt: new Date().toISOString() };
@@ -93,7 +104,7 @@ export async function addToBlocklist(handles, { purgeExisting = false } = {}) {
const incoming = (Array.isArray(handles) ? handles : [handles])
.map((h) => blocklistKey(h))
.filter(Boolean);
- const current = await readBlocklist();
+ const current = await readBlocklistForWrite();
const before = new Set(current.handles);
const next = new Set(current.handles);
const added = [];
@@ -134,7 +145,7 @@ export async function addToBlocklist(handles, { purgeExisting = false } = {}) {
export async function removeFromBlocklist(handle) {
const key = blocklistKey(handle);
if (!key) return readBlocklist();
- const current = await readBlocklist();
+ const current = await readBlocklistForWrite();
if (!current.handles.includes(key)) return current;
return writeBlocklist(current.handles.filter((h) => h !== key));
}
diff --git a/server/services/memoryClassifier.js b/server/services/memoryClassifier.js
index 31ccfb2ec4..9a33378d8b 100644
--- a/server/services/memoryClassifier.js
+++ b/server/services/memoryClassifier.js
@@ -5,11 +5,10 @@
* Falls back to pattern-based extraction if LLM is unavailable.
*/
-import { readFile } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import { getStageTemplate } from './promptService.js';
-import { atomicWrite, ensureDir, safeJSONParse, PATHS } from '../lib/fileUtils.js';
+import { atomicWrite, ensureDir, readJSONFileStrict, unreadableStoreError, PATHS } from '../lib/fileUtils.js';
import { extractJson } from '../lib/jsonExtract.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { readResponseJson } from '../lib/readResponseJson.js';
@@ -40,22 +39,23 @@ let configCache = null;
async function loadConfig() {
if (configCache) return configCache;
- if (existsSync(MEMORY_CONFIG_FILE)) {
- const content = await readFile(MEMORY_CONFIG_FILE, 'utf-8');
- // Handle empty or malformed config file
- if (content && content.trim() && content.trim().startsWith('{') && content.trim().endsWith('}')) {
- configCache = { ...DEFAULT_CONFIG, ...safeJSONParse(content, {}) };
- } else {
- console.log('⚠️ Memory classifier config file empty/malformed, using defaults');
- configCache = DEFAULT_CONFIG;
- }
- } else {
- configCache = DEFAULT_CONFIG;
+ const { ok, value } = await readJSONFileStrict(MEMORY_CONFIG_FILE, null, { allowArray: false });
+ if (!ok) {
+ // Serve defaults to readers without caching, so a repaired file is picked up.
+ console.warn('⚠️ Memory classifier config unreadable, using defaults');
+ return DEFAULT_CONFIG;
}
-
+ configCache = value ? { ...DEFAULT_CONFIG, ...value } : DEFAULT_CONFIG;
return configCache;
}
+// updateConfig refuses while the stored bytes are unreadable, so defaults
+// served to readers are never persisted over the user's config.
+async function assertConfigWritable() {
+ const { ok } = await readJSONFileStrict(MEMORY_CONFIG_FILE, null, { allowArray: false, logError: false });
+ if (!ok) throw unreadableStoreError(MEMORY_CONFIG_FILE);
+}
+
/**
* Get current configuration
*/
@@ -67,6 +67,7 @@ export async function getConfig() {
* Update configuration
*/
export async function updateConfig(updates) {
+ await assertConfigWritable();
const config = await loadConfig();
const newConfig = { ...config, ...updates };
diff --git a/server/services/messagePlaywrightSync.js b/server/services/messagePlaywrightSync.js
index 6b24e23536..2ae2dec1ee 100644
--- a/server/services/messagePlaywrightSync.js
+++ b/server/services/messagePlaywrightSync.js
@@ -1,7 +1,7 @@
import { readOutlookMessageRow, browserSummaryId, browserDetailIdentities, messageIngestionIdentity } from '../lib/messageBrowserIdentity.js';
import { join } from 'path';
import { v4 as uuidv4 } from '../lib/uuid.js';
-import { ensureDir, PATHS, safeJSONParse, tryReadFile, atomicWrite } from '../lib/fileUtils.js';
+import { ensureDir, PATHS, readJSONFileStrict, unreadableStoreError, atomicWrite } from '../lib/fileUtils.js';
import { isPlainObject } from '../lib/objects.js';
import { createMutex } from '../lib/asyncMutex.js';
import { messageLogError } from '../lib/messageLogError.js';
@@ -61,14 +61,16 @@ function outlookRowContext(selector, requireScroll = true) {
}
export async function getSelectors() {
- const content = await tryReadFile(SELECTORS_FILE);
- if (!content) return {};
- const parsed = safeJSONParse(content, {}, { context: 'messageSelectors' });
- return isPlainObject(parsed) ? parsed : {};
+ const { value } = await readJSONFileStrict(SELECTORS_FILE, {}, { allowArray: false });
+ return isPlainObject(value) ? value : {};
}
export async function updateSelectors(provider, selectors) {
- const all = await getSelectors();
+ // Display readers get {} for an unreadable file; the update refuses instead of
+ // replacing every provider's selectors with one.
+ const { ok, value } = await readJSONFileStrict(SELECTORS_FILE, {}, { allowArray: false });
+ if (!ok) throw unreadableStoreError(SELECTORS_FILE);
+ const all = isPlainObject(value) ? value : {};
all[provider] = selectors;
await ensureDir(PATHS.messages);
await atomicWrite(SELECTORS_FILE, all);
diff --git a/server/services/messagePlaywrightSync.test.js b/server/services/messagePlaywrightSync.test.js
index f20e9c8547..a009773958 100644
--- a/server/services/messagePlaywrightSync.test.js
+++ b/server/services/messagePlaywrightSync.test.js
@@ -10,7 +10,13 @@ vi.mock('./browserService.js', () => ({ findOrOpenPage, isAuthPage, evaluateOnPa
const tryReadFile = vi.fn();
vi.mock('../lib/fileUtils.js', async () => {
const actual = await vi.importActual('../lib/fileUtils.js');
- return { ...actual, tryReadFile };
+ // Selector reads are strict now; drive them from the same bytes mock.
+ const readJSONFileStrict = async (path, fallback) => {
+ const raw = await tryReadFile(path);
+ if (raw == null) return { ok: true, value: fallback };
+ try { return { ok: true, value: JSON.parse(raw) }; } catch { return { ok: false, value: fallback }; }
+ };
+ return { ...actual, tryReadFile, readJSONFileStrict };
});
const { testSelectors, syncPlaywright, sendPlaywright, refreshMessageDetail } = await import('./messagePlaywrightSync.js');
diff --git a/server/services/strictReadStoreSafety.test.js b/server/services/strictReadStoreSafety.test.js
new file mode 100644
index 0000000000..77547be5bd
--- /dev/null
+++ b/server/services/strictReadStoreSafety.test.js
@@ -0,0 +1,112 @@
+import { afterAll, afterEach, describe, expect, it, vi } from 'vitest';
+import { mkdir, readFile, rm, writeFile } from 'fs/promises';
+import { dirname, join } from 'path';
+import { makePathsProxy } from '../lib/mockPathsDataRoot.js';
+
+const disk = vi.hoisted(() => {
+ const { mkdtempSync } = require('fs');
+ const { tmpdir } = require('os');
+ const { join } = require('path');
+ return { root: mkdtempSync(join(tmpdir(), 'strict-read-store-safety-')), fault: null };
+});
+vi.mock('fs/promises', async (importOriginal) => {
+ const actual = await importOriginal();
+ return {
+ ...actual,
+ readFile: (...args) => {
+ if (args[0] === disk.fault) return Promise.reject(Object.assign(new Error('injected read fault'), { code: 'EACCES' }));
+ return actual.readFile(...args);
+ }
+ };
+});
+vi.mock('../lib/fileUtils.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ const proxy = makePathsProxy(actual, { dataRoot: disk.root, extraOverrides: {
+ root: disk.root,
+ agentPersonalities: join(disk.root, 'agents'),
+ digitalTwin: join(disk.root, 'digital-twin'),
+ messages: join(disk.root, 'messages')
+ } });
+ return new Proxy(proxy, {
+ get: (target, prop) => (prop === 'dataPath' ? (...segments) => join(disk.root, ...segments) : target[prop])
+ });
+});
+vi.mock('./pm2.js', () => ({ execPm2: vi.fn(), listProcessesStrict: vi.fn() }));
+vi.mock('./digital-twin.js', () => ({ digitalTwinEvents: { emit: vi.fn(), on: vi.fn() } }));
+vi.mock('./humanActivity.js', () => ({ deleteEvents: vi.fn(async () => ({ deleted: 0 })), listConversations: vi.fn(), sourceStats: vi.fn() }));
+vi.mock('./imessageSync.js', () => ({ getStatus: vi.fn() }));
+vi.mock('./promptService.js', () => ({ getStageTemplate: vi.fn() }));
+vi.mock('./memoryBackend.js', () => ({ getMemories: vi.fn() }));
+vi.mock('./browserService.js', () => ({
+ findOrOpenPage: vi.fn(), listCdpPages: vi.fn(), isAuthPage: vi.fn(), evaluateOnPage: vi.fn()
+}));
+
+const cases = [
+ {
+ name: 'agent drafts', path: 'agents/drafts/agent-1.json',
+ load: () => import('./agentDrafts.js'),
+ mutate: svc => svc.createDraft('agent-1', { type: 'post', content: 'New draft' }),
+ seed: [{ id: 'keep', type: 'post', content: 'Existing draft', createdAt: '2026-01-01T00:00:00.000Z' }],
+ kept: data => data.some(draft => draft.id === 'keep')
+ },
+ {
+ name: 'browser config', path: 'browser-config.json',
+ load: () => import('./browserService.js').then(() => vi.importActual('./browserService.js')),
+ mutate: svc => svc.updateConfig({ autoConnect: false }),
+ seed: { cdpHost: 'keep.example.com' },
+ kept: data => data.cdpHost === 'keep.example.com'
+ },
+ {
+ name: 'digital twin feedback', path: 'digital-twin/feedback.json',
+ load: () => import('./feedbackLoop.js'),
+ mutate: svc => svc.submitFeedback({ contentType: 'test_response', validation: 'sounds_like_me', contentSnippet: 'New' }),
+ seed: { version: '1.0.0', entries: [{ id: 'keep', validation: 'not_quite' }], stats: { totalFeedback: 1, soundsLikeMe: 0, doesntSoundLikeMe: 0 }, documentWeightAdjustments: {}, lastCalculatedAt: null },
+ kept: data => data.entries.some(entry => entry.id === 'keep')
+ },
+ {
+ name: 'iMessage blocklist', path: 'imessage-blocklist.json',
+ load: () => import('./imessageManage.js'),
+ mutate: svc => svc.addToBlocklist(['[email protected]']),
+ seed: { handles: ['[email protected]'], updatedAt: '2026-01-01T00:00:00.000Z' },
+ kept: data => data.handles.includes('[email protected]')
+ },
+ {
+ name: 'memory classifier config', path: 'memory-classifier-config.json',
+ load: () => import('./memoryClassifier.js'),
+ mutate: svc => svc.updateConfig({ minConfidence: 0.5 }),
+ seed: { keepMarker: 'user-value' },
+ kept: data => data.keepMarker === 'user-value'
+ },
+ {
+ name: 'message selectors', path: 'messages/selectors.json',
+ load: () => import('./messagePlaywrightSync.js'),
+ mutate: svc => svc.updateSelectors('teams', { messageRow: '.new' }),
+ seed: { outlook: { messageRow: '.keep' } },
+ kept: data => data.outlook.messageRow === '.keep'
+ }
+];
+
+afterEach(() => { disk.fault = null; });
+afterAll(() => rm(disk.root, { recursive: true, force: true }));
+
+describe('tolerant-read stores refuse to write defaults over unreadable bytes', () => {
+ for (const boundary of cases) {
+ it(`${boundary.name} preserves unreadable bytes, accepts the repaired file, and initializes when absent`, async () => {
+ const svc = await boundary.load();
+ const path = join(disk.root, boundary.path);
+ await mkdir(dirname(path), { recursive: true });
+ for (const bytes of ['{truncated', '', JSON.stringify(boundary.seed)]) {
+ await writeFile(path, bytes);
+ disk.fault = bytes.startsWith('{truncated') || bytes === '' ? null : path;
+ await expect(boundary.mutate(svc)).rejects.toThrow(/Unreadable JSON file/);
+ disk.fault = null;
+ expect(await readFile(path, 'utf8')).toBe(bytes);
+ }
+ await writeFile(path, JSON.stringify(boundary.seed));
+ await boundary.mutate(svc);
+ expect(boundary.kept(JSON.parse(await readFile(path, 'utf8')))).toBe(true);
+ await rm(path);
+ await expect(boundary.mutate(svc)).resolves.not.toThrow();
+ });
+ }
+});