Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: refuse writes over unreadable drafts, browser, feedback, blocklist, classifier and selector stores (#12463)
[._.]/ Adam Eivy1fd116813a8b
diff --git a/server/services/agentDrafts.js b/server/services/agentDrafts.jsindex 0f20012800..d51347560e 100644--- a/server/services/agentDrafts.js+++ b/server/services/agentDrafts.js@@ -7,7 +7,7 @@import { join } from 'path';import { v4 as uuidv4 } from '../lib/uuid.js';-import { atomicWrite, ensureDir, PATHS, safeJSONParse, tryReadFile } from '../lib/fileUtils.js';+import { atomicWrite, ensureDir, PATHS, readJSONFileStrict, unreadableStoreError } from '../lib/fileUtils.js';import { isTopLevelEntryName } from '../lib/pathSafety.js';import { ServerError } from '../lib/errorHandler.js';@@ -25,9 +25,10 @@ async function getDraftsPath(agentId) {async function loadDrafts(agentId) {const filePath = await getDraftsPath(agentId);- const content = await tryReadFile(filePath);- if (!content) return [];- return safeJSONParse(content, [], { context: `agentDrafts:${agentId}` });+ // Absent initializes to []; an unreadable file throws so no caller can save over it.+ const { ok, value } = await readJSONFileStrict(filePath, []);+ if (!ok) throw unreadableStoreError(filePath);+ return value;}async function saveDrafts(agentId, drafts) {diff --git a/server/services/agentDrafts.test.js b/server/services/agentDrafts.test.jsindex 1f6eb84a20..7ddeb93c57 100644--- a/server/services/agentDrafts.test.js+++ b/server/services/agentDrafts.test.js@@ -22,13 +22,16 @@ vi.mock('../lib/uuid.js', () => {});vi.mock('../lib/fileUtils.js', () => ({- tryReadFile: vi.fn(async (path) => fileStore.has(path) ? fileStore.get(path) : null),+ readJSONFileStrict: vi.fn(async (path, fallback) => {+ if (!fileStore.has(path)) return { ok: true, value: fallback };+ try { return { ok: true, value: JSON.parse(fileStore.get(path)) }; } catch { return { ok: false, value: fallback }; }+ }),+ unreadableStoreError: (path) => new Error(`Unreadable JSON file: ${path}`),atomicWrite: vi.fn(async (path, data) => {fileStore.set(path, typeof data === 'string' ? data : JSON.stringify(data, null, 2));}),ensureDir: vi.fn().mockResolvedValue(undefined),- PATHS: { agentPersonalities: '/mock/agents' },- safeJSONParse: (s, fallback) => { try { return JSON.parse(s); } catch { return fallback; } }+ PATHS: { agentPersonalities: '/mock/agents' }}));const { listDrafts, getDraft, createDraft, updateDraft, deleteDraft } = await import('./agentDrafts.js');diff --git a/server/services/browserService.js b/server/services/browserService.jsindex ea98825dae..6765ed75a2 100644--- a/server/services/browserService.js+++ b/server/services/browserService.js@@ -7,7 +7,7 @@import { readdir, stat, unlink } from 'fs/promises';import { join, basename, dirname, resolve, extname } from 'path';import { EventEmitter } from 'events';-import { ensureDir, safeJSONParse, PATHS, tryReadFile, atomicWrite, sleep } from '../lib/fileUtils.js';+import { ensureDir, safeJSONParse, PATHS, readJSONFileStrict, unreadableStoreError, atomicWrite, sleep } from '../lib/fileUtils.js';import { PORTS } from '../lib/ports.js';import { normalizeBrowserConfig } from '../lib/browserConfig.js';import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';@@ -58,15 +58,26 @@ async function getConfigMtimeMs() {export async function loadConfig() {const mtimeMs = await getConfigMtimeMs();if (cachedConfig && cachedConfigMtimeMs === mtimeMs) return cachedConfig;- const raw = await tryReadFile(CONFIG_FILE);- const parsed = safeJSONParse(raw, null);- cachedConfig = normalizeBrowserConfig(parsed ? { ...DEFAULT_CONFIG, ...parsed } : { ...DEFAULT_CONFIG });+ const { ok, value: parsed } = await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false });+ const config = normalizeBrowserConfig(parsed ? { ...DEFAULT_CONFIG, ...parsed } : { ...DEFAULT_CONFIG });+ // Display readers serve defaults for an unreadable file, but never cache them:+ // a repaired file must be picked up even when its mtime is unchanged.+ if (!ok) return config;+ cachedConfig = config;cachedConfigMtimeMs = mtimeMs;return cachedConfig;}+// Every write path refuses while the stored bytes are unreadable, so defaults+// served to display readers are never persisted over the real config.+async function assertConfigWritable() {+ const { ok } = await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false });+ if (!ok) throw unreadableStoreError(CONFIG_FILE);+}+export async function saveConfig(config) {await ensureDir(PATHS.data);+ await assertConfigWritable();cachedConfig = normalizeBrowserConfig({ ...DEFAULT_CONFIG, ...config });await atomicWrite(CONFIG_FILE, cachedConfig);cachedConfigMtimeMs = await getConfigMtimeMs();diff --git a/server/services/imessageManage.js b/server/services/imessageManage.jsindex ad600fe37b..e480601ef7 100644--- a/server/services/imessageManage.js+++ b/server/services/imessageManage.js@@ -6,7 +6,7 @@* remove rows from the machine-local `human_activity_events` table only.* Blocklisted handles are skipped on future syncs (see imessageSync.runSync).*/-import { dataPath, atomicWrite, tryReadFile, safeJSONParse } from '../lib/fileUtils.js';+import { dataPath, atomicWrite, readJSONFileStrict, unreadableStoreError } from '../lib/fileUtils.js';import { identityFromHandle, normalizeIdentifier, normalizePhone } from '../lib/tribeMatch.js';import * as humanActivity from './humanActivity.js';import { getStatus } from './imessageSync.js';@@ -61,9 +61,7 @@ export function blocklistKey(handle) {return raw;}-export async function readBlocklist() {- const raw = await tryReadFile(blocklistPath());- const parsed = raw ? safeJSONParse(raw, null, { allowArray: false }) : null;+function projectBlocklist(parsed) {const handles = Array.isArray(parsed?.handles)? [...new Set(parsed.handles.map((h) => blocklistKey(h)).filter(Boolean))].sort(): [];@@ -73,6 +71,19 @@ export async function readBlocklist() {};}+// Display readers (status, conversation list) serve an empty list for an+// unreadable file; mutation paths use readBlocklistForWrite and refuse.+export async function readBlocklist() {+ const { value } = await readJSONFileStrict(blocklistPath(), null, { allowArray: false });+ return projectBlocklist(value);+}++async function readBlocklistForWrite() {+ const { ok, value } = await readJSONFileStrict(blocklistPath(), null, { allowArray: false });+ if (!ok) throw unreadableStoreError(blocklistPath());+ return projectBlocklist(value);+}+async function writeBlocklist(handles) {const cleaned = [...new Set((handles || []).map((h) => blocklistKey(h)).filter(Boolean))].sort();const payload = { handles: cleaned, updatedAt: new Date().toISOString() };@@ -93,7 +104,7 @@ export async function addToBlocklist(handles, { purgeExisting = false } = {}) {const incoming = (Array.isArray(handles) ? handles : [handles]).map((h) => blocklistKey(h)).filter(Boolean);- const current = await readBlocklist();+ const current = await readBlocklistForWrite();const before = new Set(current.handles);const next = new Set(current.handles);const added = [];@@ -134,7 +145,7 @@ export async function addToBlocklist(handles, { purgeExisting = false } = {}) {export async function removeFromBlocklist(handle) {const key = blocklistKey(handle);if (!key) return readBlocklist();- const current = await readBlocklist();+ const current = await readBlocklistForWrite();if (!current.handles.includes(key)) return current;return writeBlocklist(current.handles.filter((h) => h !== key));}diff --git a/server/services/memoryClassifier.js b/server/services/memoryClassifier.jsindex 31ccfb2ec4..9a33378d8b 100644--- a/server/services/memoryClassifier.js+++ b/server/services/memoryClassifier.js@@ -5,11 +5,10 @@* Falls back to pattern-based extraction if LLM is unavailable.*/-import { readFile } from 'fs/promises';import { existsSync } from 'fs';import { join } from 'path';import { getStageTemplate } from './promptService.js';-import { atomicWrite, ensureDir, safeJSONParse, PATHS } from '../lib/fileUtils.js';+import { atomicWrite, ensureDir, readJSONFileStrict, unreadableStoreError, PATHS } from '../lib/fileUtils.js';import { extractJson } from '../lib/jsonExtract.js';import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';import { readResponseJson } from '../lib/readResponseJson.js';@@ -40,22 +39,23 @@ let configCache = null;async function loadConfig() {if (configCache) return configCache;- if (existsSync(MEMORY_CONFIG_FILE)) {- const content = await readFile(MEMORY_CONFIG_FILE, 'utf-8');- // Handle empty or malformed config file- if (content && content.trim() && content.trim().startsWith('{') && content.trim().endsWith('}')) {- configCache = { ...DEFAULT_CONFIG, ...safeJSONParse(content, {}) };- } else {- console.log('⚠️ Memory classifier config file empty/malformed, using defaults');- configCache = DEFAULT_CONFIG;- }- } else {- configCache = DEFAULT_CONFIG;+ const { ok, value } = await readJSONFileStrict(MEMORY_CONFIG_FILE, null, { allowArray: false });+ if (!ok) {+ // Serve defaults to readers without caching, so a repaired file is picked up.+ console.warn('⚠️ Memory classifier config unreadable, using defaults');+ return DEFAULT_CONFIG;}-+ configCache = value ? { ...DEFAULT_CONFIG, ...value } : DEFAULT_CONFIG;return configCache;}+// updateConfig refuses while the stored bytes are unreadable, so defaults+// served to readers are never persisted over the user's config.+async function assertConfigWritable() {+ const { ok } = await readJSONFileStrict(MEMORY_CONFIG_FILE, null, { allowArray: false, logError: false });+ if (!ok) throw unreadableStoreError(MEMORY_CONFIG_FILE);+}+/*** Get current configuration*/@@ -67,6 +67,7 @@ export async function getConfig() {* Update configuration*/export async function updateConfig(updates) {+ await assertConfigWritable();const config = await loadConfig();const newConfig = { ...config, ...updates };diff --git a/server/services/messagePlaywrightSync.js b/server/services/messagePlaywrightSync.jsindex 6b24e23536..2ae2dec1ee 100644--- a/server/services/messagePlaywrightSync.js+++ b/server/services/messagePlaywrightSync.js@@ -1,7 +1,7 @@import { readOutlookMessageRow, browserSummaryId, browserDetailIdentities, messageIngestionIdentity } from '../lib/messageBrowserIdentity.js';import { join } from 'path';import { v4 as uuidv4 } from '../lib/uuid.js';-import { ensureDir, PATHS, safeJSONParse, tryReadFile, atomicWrite } from '../lib/fileUtils.js';+import { ensureDir, PATHS, readJSONFileStrict, unreadableStoreError, atomicWrite } from '../lib/fileUtils.js';import { isPlainObject } from '../lib/objects.js';import { createMutex } from '../lib/asyncMutex.js';import { messageLogError } from '../lib/messageLogError.js';@@ -61,14 +61,16 @@ function outlookRowContext(selector, requireScroll = true) {}export async function getSelectors() {- const content = await tryReadFile(SELECTORS_FILE);- if (!content) return {};- const parsed = safeJSONParse(content, {}, { context: 'messageSelectors' });- return isPlainObject(parsed) ? parsed : {};+ const { value } = await readJSONFileStrict(SELECTORS_FILE, {}, { allowArray: false });+ return isPlainObject(value) ? value : {};}export async function updateSelectors(provider, selectors) {- const all = await getSelectors();+ // Display readers get {} for an unreadable file; the update refuses instead of+ // replacing every provider's selectors with one.+ const { ok, value } = await readJSONFileStrict(SELECTORS_FILE, {}, { allowArray: false });+ if (!ok) throw unreadableStoreError(SELECTORS_FILE);+ const all = isPlainObject(value) ? value : {};all[provider] = selectors;await ensureDir(PATHS.messages);await atomicWrite(SELECTORS_FILE, all);diff --git a/server/services/messagePlaywrightSync.test.js b/server/services/messagePlaywrightSync.test.jsindex f20e9c8547..a009773958 100644--- a/server/services/messagePlaywrightSync.test.js+++ b/server/services/messagePlaywrightSync.test.js@@ -10,7 +10,13 @@ vi.mock('./browserService.js', () => ({ findOrOpenPage, isAuthPage, evaluateOnPaconst tryReadFile = vi.fn();vi.mock('../lib/fileUtils.js', async () => {const actual = await vi.importActual('../lib/fileUtils.js');- return { ...actual, tryReadFile };+ // Selector reads are strict now; drive them from the same bytes mock.+ const readJSONFileStrict = async (path, fallback) => {+ const raw = await tryReadFile(path);+ if (raw == null) return { ok: true, value: fallback };+ try { return { ok: true, value: JSON.parse(raw) }; } catch { return { ok: false, value: fallback }; }+ };+ return { ...actual, tryReadFile, readJSONFileStrict };});const { testSelectors, syncPlaywright, sendPlaywright, refreshMessageDetail } = await import('./messagePlaywrightSync.js');diff --git a/server/services/strictReadStoreSafety.test.js b/server/services/strictReadStoreSafety.test.jsnew file mode 100644index 0000000000..77547be5bd--- /dev/null+++ b/server/services/strictReadStoreSafety.test.js@@ -0,0 +1,112 @@+import { afterAll, afterEach, describe, expect, it, vi } from 'vitest';+import { mkdir, readFile, rm, writeFile } from 'fs/promises';+import { dirname, join } from 'path';+import { makePathsProxy } from '../lib/mockPathsDataRoot.js';++const disk = vi.hoisted(() => {+ const { mkdtempSync } = require('fs');+ const { tmpdir } = require('os');+ const { join } = require('path');+ return { root: mkdtempSync(join(tmpdir(), 'strict-read-store-safety-')), fault: null };+});+vi.mock('fs/promises', async (importOriginal) => {+ const actual = await importOriginal();+ return {+ ...actual,+ readFile: (...args) => {+ if (args[0] === disk.fault) return Promise.reject(Object.assign(new Error('injected read fault'), { code: 'EACCES' }));+ return actual.readFile(...args);+ }+ };+});+vi.mock('../lib/fileUtils.js', async (importOriginal) => {+ const actual = await importOriginal();+ const proxy = makePathsProxy(actual, { dataRoot: disk.root, extraOverrides: {+ root: disk.root,+ agentPersonalities: join(disk.root, 'agents'),+ digitalTwin: join(disk.root, 'digital-twin'),+ messages: join(disk.root, 'messages')+ } });+ return new Proxy(proxy, {+ get: (target, prop) => (prop === 'dataPath' ? (...segments) => join(disk.root, ...segments) : target[prop])+ });+});+vi.mock('./pm2.js', () => ({ execPm2: vi.fn(), listProcessesStrict: vi.fn() }));+vi.mock('./digital-twin.js', () => ({ digitalTwinEvents: { emit: vi.fn(), on: vi.fn() } }));+vi.mock('./humanActivity.js', () => ({ deleteEvents: vi.fn(async () => ({ deleted: 0 })), listConversations: vi.fn(), sourceStats: vi.fn() }));+vi.mock('./imessageSync.js', () => ({ getStatus: vi.fn() }));+vi.mock('./promptService.js', () => ({ getStageTemplate: vi.fn() }));+vi.mock('./memoryBackend.js', () => ({ getMemories: vi.fn() }));+vi.mock('./browserService.js', () => ({+ findOrOpenPage: vi.fn(), listCdpPages: vi.fn(), isAuthPage: vi.fn(), evaluateOnPage: vi.fn()+}));++const cases = [+ {+ name: 'agent drafts', path: 'agents/drafts/agent-1.json',+ load: () => import('./agentDrafts.js'),+ mutate: svc => svc.createDraft('agent-1', { type: 'post', content: 'New draft' }),+ seed: [{ id: 'keep', type: 'post', content: 'Existing draft', createdAt: '2026-01-01T00:00:00.000Z' }],+ kept: data => data.some(draft => draft.id === 'keep')+ },+ {+ name: 'browser config', path: 'browser-config.json',+ load: () => import('./browserService.js').then(() => vi.importActual('./browserService.js')),+ mutate: svc => svc.updateConfig({ autoConnect: false }),+ seed: { cdpHost: 'keep.example.com' },+ kept: data => data.cdpHost === 'keep.example.com'+ },+ {+ name: 'digital twin feedback', path: 'digital-twin/feedback.json',+ load: () => import('./feedbackLoop.js'),+ mutate: svc => svc.submitFeedback({ contentType: 'test_response', validation: 'sounds_like_me', contentSnippet: 'New' }),+ seed: { version: '1.0.0', entries: [{ id: 'keep', validation: 'not_quite' }], stats: { totalFeedback: 1, soundsLikeMe: 0, doesntSoundLikeMe: 0 }, documentWeightAdjustments: {}, lastCalculatedAt: null },+ kept: data => data.entries.some(entry => entry.id === 'keep')+ },+ {+ name: 'iMessage blocklist', path: 'imessage-blocklist.json',+ load: () => import('./imessageManage.js'),+ mutate: svc => svc.addToBlocklist(['[email protected]']),+ seed: { handles: ['[email protected]'], updatedAt: '2026-01-01T00:00:00.000Z' },+ kept: data => data.handles.includes('[email protected]')+ },+ {+ name: 'memory classifier config', path: 'memory-classifier-config.json',+ load: () => import('./memoryClassifier.js'),+ mutate: svc => svc.updateConfig({ minConfidence: 0.5 }),+ seed: { keepMarker: 'user-value' },+ kept: data => data.keepMarker === 'user-value'+ },+ {+ name: 'message selectors', path: 'messages/selectors.json',+ load: () => import('./messagePlaywrightSync.js'),+ mutate: svc => svc.updateSelectors('teams', { messageRow: '.new' }),+ seed: { outlook: { messageRow: '.keep' } },+ kept: data => data.outlook.messageRow === '.keep'+ }+];++afterEach(() => { disk.fault = null; });+afterAll(() => rm(disk.root, { recursive: true, force: true }));++describe('tolerant-read stores refuse to write defaults over unreadable bytes', () => {+ for (const boundary of cases) {+ it(`${boundary.name} preserves unreadable bytes, accepts the repaired file, and initializes when absent`, async () => {+ const svc = await boundary.load();+ const path = join(disk.root, boundary.path);+ await mkdir(dirname(path), { recursive: true });+ for (const bytes of ['{truncated', '', JSON.stringify(boundary.seed)]) {+ await writeFile(path, bytes);+ disk.fault = bytes.startsWith('{truncated') || bytes === '' ? null : path;+ await expect(boundary.mutate(svc)).rejects.toThrow(/Unreadable JSON file/);+ disk.fault = null;+ expect(await readFile(path, 'utf8')).toBe(bytes);+ }+ await writeFile(path, JSON.stringify(boundary.seed));+ await boundary.mutate(svc);+ expect(boundary.kept(JSON.parse(await readFile(path, 'utf8')))).toBe(true);+ await rm(path);+ await expect(boundary.mutate(svc)).resolves.not.toThrow();+ });+ }+});